diff --git "a/weights/David-partial_shared-deep_efficiency/20251012_181608/best_model_acc83.66_metadata.json" "b/weights/David-partial_shared-deep_efficiency/20251012_181608/best_model_acc83.66_metadata.json" new file mode 100644--- /dev/null +++ "b/weights/David-partial_shared-deep_efficiency/20251012_181608/best_model_acc83.66_metadata.json" @@ -0,0 +1,778 @@ +{ + "epoch": 0, + "optimizer_state_dict": { + "state": { + "0": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 9.5899e-05, 9.0451e-05, -1.3394e-04, ..., 4.4801e-06,\n 3.6685e-05, -1.1002e-04],\n [ 5.9246e-04, 3.1149e-04, -1.1248e-04, ..., -1.3479e-04,\n -2.8052e-05, -2.0691e-05],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n ...,\n [-5.4243e-05, -6.6344e-05, -1.7443e-05, ..., -1.5093e-05,\n 1.0849e-06, -2.4451e-04],\n [-1.3069e-04, 2.1378e-05, -1.8115e-04, ..., 8.3143e-05,\n 7.5852e-04, 1.0524e-04],\n [-5.3249e-44, 2.1019e-44, -5.0447e-44, ..., -4.2039e-44,\n 1.8217e-44, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7455e-07, 1.5473e-07, 1.6438e-07, ..., 1.8849e-07, 3.9566e-07,\n 1.6343e-07],\n [2.8288e-07, 2.3531e-07, 2.4554e-07, ..., 2.9762e-07, 5.6862e-07,\n 2.8408e-07],\n [4.2580e-13, 2.9175e-11, 1.8164e-11, ..., 1.1982e-10, 2.6852e-11,\n 1.6025e-11],\n ...,\n [2.6640e-07, 2.4480e-07, 2.6248e-07, ..., 2.5918e-07, 8.0693e-07,\n 3.9353e-07],\n [2.9364e-07, 2.4105e-07, 2.6777e-07, ..., 4.7707e-07, 6.3455e-07,\n 2.7318e-07],\n [3.3648e-11, 2.5780e-11, 1.6851e-10, ..., 1.1837e-10, 9.6041e-10,\n 3.0462e-11]], device='cuda:0')" + }, + "1": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-9.7592e-03, 1.1319e-02, 5.6052e-45, ..., 6.7142e-03,\n 1.9028e-02, 1.3298e-42], device='cuda:0')", + "exp_avg_sq": "tensor([5.4014e-04, 6.8839e-04, 3.5959e-07, ..., 9.2684e-04, 8.3842e-04,\n 1.5644e-07], device='cuda:0')" + }, + "2": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-1.5880e-06, -3.1759e-08, -5.6052e-45, ..., 4.1000e-06,\n 4.8767e-07, 5.6052e-45],\n [ 1.7498e-06, 3.1138e-06, 5.6052e-45, ..., -5.4318e-06,\n 1.3745e-05, 5.6052e-45],\n [-3.7676e-06, -1.0867e-05, -5.6052e-45, ..., 4.3050e-06,\n 1.3417e-05, -5.6052e-45],\n ...,\n [-2.9452e-06, -3.2398e-05, 5.6052e-45, ..., -3.5035e-06,\n 3.3135e-05, 5.6052e-45],\n [-3.5966e-06, 2.3772e-05, -5.6052e-45, ..., -5.5989e-06,\n -1.6322e-05, 5.6052e-45],\n [ 3.9226e-06, 1.1398e-04, -5.6052e-45, ..., 6.4996e-06,\n 1.8231e-05, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.4595e-10, 6.2769e-10, 8.8863e-12, ..., 7.6261e-10, 1.9069e-09,\n 1.4589e-10],\n [1.0867e-08, 1.4159e-09, 2.9783e-13, ..., 3.0476e-09, 3.1205e-08,\n 5.2019e-12],\n [3.2086e-09, 1.8066e-08, 6.3814e-14, ..., 1.7751e-08, 6.0463e-09,\n 2.6489e-11],\n ...,\n [1.6759e-08, 2.2370e-08, 4.8316e-12, ..., 3.8965e-08, 2.0533e-08,\n 7.6158e-11],\n [1.1110e-08, 1.4160e-08, 7.0680e-13, ..., 2.7756e-08, 1.0363e-08,\n 1.4726e-10],\n [1.1486e-08, 1.7880e-08, 2.7311e-13, ..., 1.9126e-08, 1.6811e-08,\n 2.2708e-12]], device='cuda:0')" + }, + "3": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.0223e-03, -6.2250e-04, 1.1932e-04, 2.7359e-03, 3.2055e-04,\n 8.8993e-04, -6.4894e-04, 1.4584e-03, -1.6458e-03, -5.6411e-06,\n -6.5042e-04, -1.3630e-03, 4.4974e-03, -5.3840e-04, 3.2298e-04,\n 1.7032e-16, 1.8259e-03, 9.6576e-05, -7.2051e-04, -8.0842e-04,\n 3.6554e-04, -2.4645e-03, 1.7423e-03, 1.6227e-04, 4.0884e-04,\n -2.2534e-04, 1.6778e-03, 2.1608e-04, -2.7606e-04, -8.9818e-04,\n 3.0565e-04, -1.0632e-03, 4.2780e-03, -4.4089e-04, -3.2698e-04,\n 7.5978e-04, 5.8960e-04, -2.8925e-03, 1.0621e-03, -4.8369e-04,\n 1.1905e-03, -5.8391e-04, 3.3380e-04, -2.1417e-03, 3.5795e-03,\n -2.9696e-03, 1.8169e-04, 1.4141e-03, -6.3706e-06, 1.4825e-03,\n 1.6731e-23, 3.2230e-03, 1.2457e-03, -7.3842e-04, -5.4798e-04,\n -1.2885e-03, 4.1982e-04, 2.5118e-04, -4.9653e-04, 2.2669e-03,\n -9.9582e-04, 1.0889e-03, 4.8646e-04, 6.0899e-04, -3.1982e-04,\n 1.3704e-04, -1.9144e-04, -2.5652e-04, 1.9308e-03, 9.7491e-04,\n -1.5235e-03, 2.0898e-03, -6.6170e-05, -2.9974e-04, 1.5166e-03,\n 8.7260e-04, 6.9008e-04, 2.0239e-03, 1.4168e-03, 2.0021e-03,\n 2.5394e-03, -6.5992e-04, -1.9072e-03, 2.5381e-03, 2.6772e-03,\n 5.8064e-04, -1.9916e-05, 4.1941e-04, 1.5490e-03, -9.0543e-04,\n 6.4467e-04, 4.3931e-04, 7.1908e-04, 2.4234e-04, -1.5602e-03,\n -1.3681e-03, 9.3314e-04, -6.5354e-06, -1.8165e-03, -1.0966e-03,\n 2.3448e-06, 1.0385e-03, 9.8903e-04, -9.9776e-04, -4.6524e-04,\n 5.1754e-04, -8.5756e-04, -3.1468e-04, -9.1784e-04, 2.8934e-04,\n -1.6319e-03, -1.6519e-03, -3.9491e-04, 2.2883e-05, 9.4969e-04,\n 1.1408e-03, 9.0629e-05, -7.8423e-04, -9.1533e-04, -1.0041e-03,\n 1.0388e-03, 7.6393e-04, 5.7447e-04, 3.2073e-04, 9.7273e-04,\n -6.8513e-04, 7.1452e-04, 3.2146e-04, 1.4605e-03, -2.0282e-03,\n -2.4722e-03, 7.7496e-04, 1.7264e-05, 8.8381e-04, -1.9730e-04,\n 3.0501e-04, -1.1153e-03, 2.9391e-04, -5.8613e-04, -3.3163e-05,\n -9.4068e-04, -1.5372e-03, -3.9896e-04, 9.8883e-04, 3.4162e-03,\n -1.1896e-03, -6.5575e-04, 2.2250e-03, 2.5805e-03, -6.5859e-04,\n -4.7473e-03, -7.9714e-04, 6.5098e-04, -6.9681e-05, -4.0783e-03,\n 1.3146e-03, -1.4359e-03, 4.6094e-04, -2.2336e-03, 2.0778e-04,\n 1.9145e-03, -6.0671e-04, 5.5170e-06, -2.6494e-03, 2.0343e-05,\n -3.5601e-04, 1.8816e-03, 7.5519e-04, 1.1207e-03, -2.1777e-03,\n -1.4198e-04, 3.7269e-04, -9.3666e-04, 2.9973e-04, -7.4203e-04,\n -4.7579e-04, -8.1279e-05, -3.9510e-04, 4.5184e-11, -1.1884e-03,\n 1.9452e-04, 1.4376e-03, 8.0849e-05, 1.1180e-04, 3.8096e-03,\n 3.3149e-03, 2.1441e-03, -3.8321e-06, -3.1289e-03, -1.6129e-03,\n -2.8424e-03, -1.8631e-03, -1.8647e-04, 3.0892e-04, 2.0043e-04,\n -2.2066e-03, -3.4477e-07, -6.2800e-04, 6.6904e-04, 1.4501e-03,\n -1.7647e-03, -4.6952e-04, 5.2781e-04, 6.2486e-04, -1.3570e-03,\n 8.8856e-04, 8.5368e-05, -9.6612e-04, 4.2372e-03, 2.7845e-03,\n -4.7712e-04, -9.5370e-04, 1.5701e-04, -2.3039e-04, 1.8631e-03,\n 1.6802e-03, 1.1062e-03, -4.4999e-11, -9.1722e-04, -5.9495e-04,\n -9.8616e-04, 5.7328e-04, 5.4255e-05, -1.2166e-03, 2.1598e-03,\n -1.7361e-03, 4.2048e-23, 1.3808e-03, 1.6579e-03, 1.0670e-03,\n -2.9993e-04, -4.5546e-04, 5.2201e-04, -1.9753e-04, -1.3091e-03,\n 3.5099e-08, 1.3277e-03, 5.1924e-03, 5.1628e-05, -4.8185e-05,\n 3.2388e-04, 2.8906e-03, 2.4560e-03, 1.2489e-04, 7.2627e-13,\n -1.1925e-03, -8.5215e-04, 2.4509e-04, 4.0742e-04, -2.3896e-04,\n -2.2694e-03, 4.7032e-05, 3.8562e-03, -3.0305e-03, -6.6765e-05,\n 1.6681e-03, -5.8948e-04, 1.6479e-03, -1.9175e-03, -9.7945e-04,\n 2.4784e-03, 5.6052e-45, -1.2805e-04, 1.0002e-03, -1.6383e-03,\n -6.9470e-04, 1.5307e-03, -9.9650e-04, 1.0900e-03, 1.3464e-03,\n 7.4622e-07, 1.9653e-03, 1.1921e-03, 2.7871e-03, -6.6374e-04,\n 8.7590e-05, -1.8877e-03, -1.2237e-03, -9.6398e-05, -3.7533e-04,\n -1.1945e-03, -1.4115e-03, -3.0891e-03, 5.7144e-03, -1.2635e-03,\n -1.7020e-03, 1.5207e-03, 1.5883e-03, 5.6052e-45, 7.2957e-04,\n 8.6090e-04, 1.6985e-04, -1.5497e-05, 2.0792e-03, 1.6059e-03,\n 8.1164e-04, -1.1511e-03, 3.6904e-03, 1.0420e-03, -2.0619e-03,\n -1.0862e-04, 2.9092e-03, -5.5706e-04, -2.5742e-03, 1.8810e-03,\n -9.0256e-04, 2.7485e-03, 3.8224e-04, -4.9043e-03, 1.8857e-03,\n 1.1503e-04, 1.1653e-03, 3.4519e-03, -3.9849e-04, 2.2785e-04,\n -4.2298e-04, -1.6702e-03, -1.2481e-03, -1.3394e-03, -1.4226e-03,\n -3.8849e-04, 1.4937e-03, 3.2975e-05, 6.3518e-04, 6.0594e-04,\n 1.9148e-04, 4.7301e-04, -1.8720e-03, -1.6405e-03, -1.3852e-03,\n -3.0771e-03, 2.0300e-04, -1.1182e-03, 1.4648e-04, 1.0196e-03,\n 1.2286e-03, -4.4019e-04, -3.1594e-05, -3.9059e-03, -2.3955e-03,\n 1.4597e-03, -1.1090e-03, 1.6045e-04, 2.8725e-03, -4.8503e-04,\n -2.2554e-04, 5.1380e-04, 1.4364e-03, 8.9967e-05, -2.7166e-04,\n -1.3709e-03, 8.4219e-05, 2.9554e-04, -2.2994e-03, 1.8653e-03,\n 1.7077e-03, -7.6643e-04, 5.1666e-11, 1.2127e-03, 8.4725e-04,\n -1.7219e-03, 1.3750e-04, -3.5718e-12, 1.5684e-09, -1.0333e-03,\n -3.2165e-03, -9.2555e-04, 7.3100e-16, -2.3838e-03, -4.0852e-04,\n 8.6289e-04, -3.5978e-06, -5.9951e-04, -2.0326e-03, -1.1507e-03,\n -1.2068e-03, 1.2202e-03, -5.7621e-05, 1.2909e-04, 2.2052e-03,\n -1.0017e-03, 2.9398e-05, 1.0399e-04, -4.4317e-03, -1.0184e-03,\n 2.2000e-43, -6.3520e-04, -2.1255e-03, -4.1787e-05, 1.9095e-03,\n -2.1959e-03, -7.3642e-04, 1.5396e-03, 3.5853e-03, 2.6070e-03,\n -2.5867e-03, 3.2624e-04, 1.6875e-04, 3.7484e-03, 1.8574e-03,\n -4.6336e-04, 9.4597e-04, -4.5105e-04, 2.1868e-03, -2.7116e-04,\n 1.0499e-04, -2.7101e-04, 3.5166e-04, -6.7723e-04, 2.5759e-04,\n -2.6947e-03, -3.9494e-03, 1.5208e-03, -6.9738e-04, 2.1797e-03,\n -1.1773e-04, -2.2966e-03, 9.0962e-04, -1.2016e-03, -9.6632e-04,\n -1.0348e-03, 9.1066e-04, 1.3878e-03, -3.5105e-05, 1.1922e-03,\n 1.3371e-03, -5.3324e-04, -7.2900e-04, 1.0239e-03, 7.4940e-04,\n -1.7041e-03, -1.7036e-03, 1.3978e-04, 2.3456e-03, 1.0473e-04,\n 1.8614e-10, -1.2293e-04, -6.7521e-05, -2.3571e-03, -1.3675e-03,\n -9.2255e-04, -1.4252e-03, 1.2698e-03, 3.0670e-04, 9.5333e-04,\n -2.1228e-04, -2.0223e-03, -1.0631e-03, 7.8936e-25, 9.7492e-04,\n 1.0492e-04, -1.4349e-03, -9.4759e-04, -3.3608e-03, -1.4989e-03,\n 1.3950e-03, -1.3999e-03, 6.6500e-30, 4.8085e-04, -6.5462e-04,\n 1.4268e-03, -7.1574e-04, 5.4885e-04, -3.3218e-03, 1.4525e-04,\n 1.8167e-03, 1.1673e-03, 2.9934e-03, -5.1324e-03, 1.3037e-03,\n 6.2157e-04, -8.6576e-04, 9.6651e-04, 1.3898e-03, 2.4715e-03,\n 3.7215e-03, 1.2441e-03, -9.5008e-04, 6.6562e-04, 2.3222e-03,\n 1.0439e-03, 1.1843e-03, 8.4510e-04, -1.2217e-04, -1.7402e-04,\n 1.7006e-03, -2.0344e-03, 1.2531e-03, 3.3188e-03, 2.4970e-04,\n 4.9202e-04, -4.1824e-04, 5.8190e-04, 5.6603e-04, -1.1790e-03,\n 6.1909e-42, -1.4746e-03, -7.1667e-04, -5.1673e-04, -5.6730e-04,\n 3.2731e-03, 1.1211e-03, 2.4636e-03, -2.2061e-03, 2.1560e-04,\n -5.3094e-04, 4.0541e-04, 6.8727e-04, -1.0111e-03, -3.0332e-03,\n -4.7639e-04, 2.0896e-04, -2.8033e-06, 5.1751e-04, 2.2549e-03,\n -1.6131e-03, 3.5561e-04, 7.7300e-04, 5.5826e-04, 8.7780e-04,\n 2.3890e-03, 3.4646e-04, -6.1948e-05, 1.6031e-03, 7.2774e-04,\n -7.5880e-04, -9.9257e-05, -9.6350e-04, -1.4124e-03, 2.8227e-04,\n 1.5068e-03, -7.7068e-06, 2.4511e-03, 5.0621e-04, 5.4739e-04,\n -2.2444e-03, -7.3666e-04, -1.9130e-03, 1.1012e-03, -3.9591e-04,\n 3.7749e-03, 3.2644e-03, -8.1866e-04, -2.4747e-04, 4.3293e-04,\n -4.6883e-04, -1.1849e-03, 1.7423e-03, 1.4717e-37, 1.0413e-03,\n -3.4514e-04, 8.3113e-20, 2.0109e-03, -2.6494e-03, 2.6385e-03,\n -1.5222e-04, -1.5866e-03, -1.8369e-04, 1.0157e-03, -1.3997e-03,\n -2.3679e-03, 5.2197e-04, -1.5813e-03, 1.2115e-03, 2.4857e-03,\n 1.0211e-03, 3.4768e-04, 1.0766e-03, 1.7815e-04, -7.2486e-04,\n 1.0419e-05, -2.8715e-03, 1.4659e-03, -1.1406e-04, -1.8320e-03,\n 1.2399e-03, 1.8177e-03, 1.8036e-03, -1.3509e-03, -5.6943e-04,\n -6.4401e-04, -2.4908e-03, -7.8927e-04, 2.6222e-03, 9.7938e-04,\n 6.0840e-04, -1.5986e-03, -6.3500e-04, 3.3424e-03, 1.8084e-03,\n 9.3119e-04, 6.1832e-04, 2.6678e-03, -7.5178e-04, 1.9433e-03,\n 1.4067e-03, -2.0986e-03, 5.8382e-14, -2.5384e-03, 5.7768e-39,\n -7.5676e-04, -9.4125e-04, 1.8404e-03, -2.1408e-03, 6.9605e-04,\n -1.2034e-03, -2.6959e-03, -1.9886e-03, 5.3700e-04, 2.4800e-03,\n 1.6054e-03, 5.8097e-06, -2.0238e-03, -1.3444e-03, -1.6246e-03,\n 7.5850e-04, -1.8206e-04, -1.2746e-03, 1.1897e-03, -2.1837e-04,\n -4.0576e-04, -8.7991e-04, 4.6731e-04, 3.4580e-03, -8.2541e-04,\n 2.4841e-03, -1.5547e-03, -1.0109e-04, -2.5026e-05, -6.6090e-04,\n -4.1738e-04, 4.7301e-04, 2.1520e-04, -1.4948e-03, -1.2801e-03,\n 9.0009e-05, -7.6540e-04, 1.9966e-03, -1.5618e-03, -1.8660e-03,\n 3.1184e-03, -3.1104e-04, -6.9454e-04, -8.6846e-05, -1.8557e-03,\n -3.2563e-04, 1.8086e-03, -2.2494e-03, -5.1844e-04, -1.3806e-04,\n -2.7696e-03, 1.3534e-03, 1.1383e-03, 2.6167e-04, 1.2346e-03,\n 1.9530e-03, -1.8854e-03, -6.1367e-04, -7.9860e-04, 1.4750e-03,\n 1.0850e-05, -4.3176e-04, 7.4458e-15, 3.8624e-03, -1.4660e-04,\n 4.0854e-04, -1.2117e-03, -3.4482e-03, 2.0973e-03, -2.4438e-03,\n 5.6052e-45, -1.6332e-04, -3.2675e-04, 4.6658e-05, -1.1664e-03,\n -1.6726e-03, 6.0468e-04, -2.2714e-03, -1.2337e-03, 9.3529e-04,\n 8.1050e-04, 2.0860e-04, 1.1143e-03, -4.2081e-04, 1.9554e-04,\n 2.7405e-18, 2.0691e-03, -2.6632e-04, 9.4454e-04, -5.8184e-04,\n 2.4757e-04, 2.9429e-04, 1.1312e-03, 1.1239e-03, 5.8862e-04,\n -8.2111e-03, 2.8919e-03, 8.3653e-04, 1.6335e-04, -3.1795e-04,\n -2.0204e-03, 3.1888e-03, -1.1729e-03, -1.9777e-04, 1.2217e-03,\n -2.8604e-03, -2.5148e-04, -8.9109e-04, 2.9445e-03, -2.7045e-03,\n 2.4740e-04, 8.8607e-04, -9.9198e-04, 3.5817e-04, -7.9303e-04,\n 2.9244e-04, -1.4646e-03, -2.2806e-03, -3.8021e-03, 5.0627e-05,\n -1.0697e-03, -1.1091e-03, 1.5051e-03, 2.2163e-03, 7.1514e-04,\n 9.1761e-04, -1.6599e-03, 2.1353e-03, -2.8234e-03, 1.7833e-03,\n 1.1468e-03, 6.5095e-07, 1.3421e-04, 1.7000e-03, 1.4874e-03,\n 5.4241e-04, 6.3298e-05, -2.6652e-04, 3.3391e-04, -1.4755e-03,\n -1.6124e-03, 5.6052e-45, -2.3023e-03, 1.0539e-03, -2.3832e-04,\n -1.5225e-03, 1.8926e-03, -3.7820e-03, -2.8102e-03, 5.4682e-04,\n -2.1806e-03, 3.1369e-03, 3.5329e-20, 3.6601e-07, 1.7263e-03,\n 1.1390e-04, -4.0792e-04, 2.8897e-04, 6.0026e-08, 1.5154e-04,\n -1.9084e-03, 2.0053e-04, 5.4283e-04, -1.7294e-03, -3.2887e-04,\n 1.0002e-03, -7.6420e-04, -1.1349e-03], device='cuda:0')", + "exp_avg_sq": "tensor([2.6745e-05, 5.7427e-05, 1.7244e-04, 2.4014e-04, 2.0346e-04, 3.5612e-04,\n 4.9006e-04, 3.8887e-05, 3.1793e-04, 5.7759e-04, 3.4492e-04, 4.1601e-04,\n 1.1809e-04, 8.1672e-05, 2.5869e-04, 1.5750e-10, 2.3157e-04, 1.9779e-04,\n 5.0671e-04, 2.7343e-04, 3.4466e-04, 1.2805e-04, 4.8012e-04, 9.1349e-05,\n 3.7917e-04, 1.6710e-06, 4.1065e-04, 2.9670e-04, 3.9578e-04, 1.2543e-04,\n 2.9090e-04, 3.3188e-05, 5.5040e-04, 3.3055e-05, 2.4933e-04, 3.5596e-04,\n 4.5225e-04, 6.3822e-04, 3.2569e-04, 4.7059e-04, 3.2295e-04, 4.7646e-04,\n 3.3339e-04, 5.6706e-04, 3.0006e-05, 4.1568e-05, 5.6610e-04, 2.7726e-04,\n 7.0777e-04, 1.8031e-04, 4.2145e-08, 4.7882e-04, 1.0650e-04, 5.4585e-04,\n 4.9975e-04, 4.0518e-04, 3.8739e-06, 2.7290e-05, 4.1485e-04, 2.2712e-04,\n 6.5704e-04, 1.9587e-05, 4.1758e-04, 4.9234e-04, 1.6227e-04, 3.4776e-04,\n 2.8933e-04, 2.2729e-04, 3.3568e-04, 1.8601e-04, 5.8433e-04, 8.1805e-04,\n 3.9822e-04, 5.2834e-06, 3.7247e-04, 3.2403e-04, 1.0865e-04, 1.9102e-05,\n 3.0660e-04, 3.0496e-04, 4.4028e-04, 1.2135e-04, 3.5435e-04, 3.2343e-04,\n 5.0040e-04, 3.5487e-04, 2.8742e-04, 3.1324e-04, 3.4272e-05, 4.3253e-04,\n 3.7912e-04, 3.4236e-04, 1.6786e-04, 4.0532e-04, 6.0231e-04, 3.6897e-04,\n 3.8686e-05, 2.1214e-07, 2.1447e-04, 3.1626e-04, 3.1828e-07, 9.3292e-05,\n 3.2392e-04, 2.7838e-04, 3.1685e-04, 2.9281e-04, 3.9728e-04, 2.3866e-04,\n 5.4385e-04, 1.7615e-05, 3.2678e-04, 4.8936e-04, 3.1949e-04, 1.0105e-04,\n 4.2100e-04, 5.0161e-04, 2.8933e-04, 8.1885e-04, 3.2394e-04, 1.0094e-04,\n 2.2512e-06, 2.3202e-04, 4.1342e-04, 2.0012e-04, 2.8014e-04, 1.1324e-04,\n 4.9391e-05, 2.7475e-04, 2.6570e-06, 5.1276e-04, 2.9600e-04, 3.7265e-04,\n 1.5811e-04, 3.2074e-04, 3.0090e-04, 3.3863e-04, 7.7642e-04, 5.4690e-04,\n 1.1520e-04, 1.8269e-05, 5.1430e-04, 1.4087e-04, 4.1233e-04, 3.6385e-04,\n 5.6178e-04, 4.3178e-05, 3.5226e-04, 3.1806e-04, 3.3728e-04, 3.4109e-04,\n 1.3938e-04, 5.4420e-04, 5.7503e-04, 8.1749e-05, 3.3712e-04, 1.6776e-04,\n 5.4599e-04, 5.6778e-04, 3.4861e-04, 3.6539e-04, 7.4241e-05, 3.5415e-04,\n 9.1054e-06, 2.5324e-05, 9.4989e-07, 2.1214e-04, 3.9906e-04, 3.9321e-04,\n 4.4481e-05, 2.1854e-04, 7.8544e-05, 2.7012e-04, 5.3350e-04, 2.5277e-04,\n 4.8010e-04, 1.3029e-04, 1.7131e-05, 3.8508e-05, 2.1290e-08, 4.8881e-04,\n 2.2809e-04, 2.7456e-04, 4.4980e-04, 9.3950e-06, 5.6544e-04, 2.2376e-04,\n 8.4049e-05, 1.1415e-05, 4.6994e-04, 3.6342e-04, 4.0211e-04, 2.8115e-04,\n 4.1728e-04, 6.3749e-06, 5.3925e-04, 2.6218e-04, 4.4404e-07, 2.6290e-04,\n 3.2087e-04, 3.2490e-04, 2.9465e-04, 4.6664e-06, 2.6678e-05, 3.3106e-06,\n 2.3696e-04, 3.0719e-05, 8.1106e-06, 3.8169e-04, 4.6252e-04, 4.4306e-04,\n 3.2499e-04, 3.5814e-06, 4.1192e-04, 3.6956e-04, 4.3479e-04, 3.6977e-04,\n 3.1770e-04, 2.9737e-06, 3.8338e-04, 1.4744e-05, 2.6068e-04, 1.6280e-04,\n 4.0236e-04, 2.0972e-04, 2.2624e-04, 2.0540e-04, 4.4606e-08, 3.6017e-04,\n 4.2896e-04, 1.8151e-04, 2.9302e-04, 2.6086e-04, 2.4430e-04, 4.7971e-04,\n 1.9654e-04, 4.8439e-07, 3.6540e-04, 4.6699e-04, 2.4675e-05, 5.1459e-04,\n 7.3179e-05, 1.1241e-04, 5.4044e-04, 8.0347e-06, 3.3534e-08, 3.7320e-04,\n 3.9754e-04, 2.0821e-04, 3.2812e-04, 2.2013e-05, 4.1182e-04, 1.9260e-04,\n 2.7250e-04, 7.9991e-05, 3.4825e-04, 3.5724e-04, 3.8134e-04, 2.1162e-04,\n 2.3396e-04, 1.8171e-04, 2.4521e-04, 1.6520e-08, 1.4226e-06, 2.9941e-04,\n 2.4449e-04, 5.1514e-04, 3.4691e-04, 5.6906e-04, 2.9427e-04, 2.2747e-04,\n 3.2165e-04, 4.6482e-04, 2.7519e-04, 3.6254e-04, 9.1739e-06, 7.1223e-05,\n 8.2866e-04, 4.3646e-04, 7.6486e-05, 4.4174e-04, 2.1155e-04, 1.3589e-05,\n 1.8081e-05, 5.5879e-04, 1.8204e-04, 4.5086e-04, 4.5728e-04, 2.8846e-04,\n 1.3197e-07, 1.9750e-04, 2.8149e-04, 3.3278e-04, 1.0762e-03, 2.0881e-04,\n 3.7253e-04, 3.4967e-04, 2.8545e-04, 2.3098e-04, 5.6294e-04, 7.6243e-05,\n 1.5560e-04, 4.9813e-04, 4.3048e-04, 3.0373e-04, 3.7999e-04, 1.6136e-04,\n 5.6018e-04, 4.1815e-04, 1.1671e-04, 8.4332e-05, 2.7816e-07, 4.6575e-04,\n 4.5744e-04, 4.4994e-04, 3.5221e-04, 3.7742e-04, 2.8488e-04, 2.1219e-04,\n 1.0669e-04, 2.5418e-04, 1.0511e-05, 3.7827e-04, 1.8807e-04, 3.8720e-04,\n 3.4537e-04, 5.0445e-04, 2.7301e-04, 3.8978e-04, 1.8614e-04, 1.2854e-04,\n 2.0974e-04, 3.8571e-04, 1.9316e-04, 3.5491e-04, 2.1645e-04, 4.2235e-04,\n 4.3602e-04, 3.2581e-04, 4.1128e-04, 5.3918e-04, 2.6874e-05, 3.3469e-04,\n 1.0433e-05, 2.7474e-04, 4.8627e-04, 4.4440e-04, 2.3542e-04, 5.0310e-04,\n 1.4290e-05, 3.1983e-04, 2.5207e-05, 1.7365e-05, 7.0564e-04, 8.0257e-05,\n 3.0777e-04, 3.4789e-04, 2.1089e-04, 1.1064e-08, 3.6470e-04, 4.8767e-04,\n 3.8482e-04, 2.2779e-05, 7.9611e-07, 1.2865e-06, 2.2276e-04, 1.4671e-04,\n 2.7998e-04, 5.5567e-08, 3.2815e-04, 1.9038e-04, 1.9460e-04, 4.2730e-04,\n 3.3574e-04, 1.8898e-04, 1.3579e-04, 7.0002e-05, 2.4946e-04, 4.9098e-07,\n 3.5248e-04, 5.3029e-04, 4.8497e-05, 2.3178e-04, 2.4949e-05, 5.4094e-05,\n 5.0355e-04, 1.2479e-08, 3.6264e-04, 4.2785e-04, 2.7851e-04, 3.5105e-04,\n 3.1660e-04, 4.9506e-04, 5.4886e-04, 3.0533e-04, 1.4974e-04, 9.7898e-05,\n 2.7041e-04, 3.1786e-05, 1.6124e-04, 3.6245e-04, 3.4248e-04, 6.4621e-04,\n 4.6383e-04, 1.5977e-05, 1.9849e-04, 1.8266e-04, 1.6311e-04, 3.1548e-05,\n 2.4598e-04, 4.3774e-04, 2.2370e-04, 7.5739e-05, 1.7536e-04, 6.7410e-04,\n 5.8336e-04, 8.3936e-05, 2.8434e-04, 4.8647e-04, 3.9174e-04, 4.3222e-04,\n 3.5297e-04, 1.9705e-04, 2.1004e-04, 1.3280e-05, 5.0403e-04, 4.5786e-04,\n 2.4535e-04, 1.5910e-04, 5.2790e-04, 3.3431e-04, 2.7171e-04, 2.3559e-04,\n 8.8642e-07, 6.0294e-04, 7.0795e-05, 3.2576e-08, 2.6142e-04, 2.9013e-04,\n 2.5731e-04, 3.3795e-04, 3.2748e-04, 4.5852e-04, 4.3934e-04, 3.8045e-05,\n 3.8635e-05, 3.3039e-04, 6.3513e-04, 2.7716e-04, 5.2130e-08, 1.0275e-04,\n 1.8716e-05, 1.0791e-04, 2.9250e-04, 3.3586e-04, 2.0214e-04, 3.6633e-04,\n 3.7347e-04, 3.5879e-08, 3.2088e-04, 1.6725e-05, 4.0837e-04, 1.4550e-04,\n 1.6732e-04, 3.4296e-04, 9.3075e-05, 2.3618e-04, 5.1267e-04, 3.2324e-04,\n 2.1001e-05, 2.9118e-05, 3.2107e-04, 2.5072e-04, 1.9531e-07, 5.0002e-04,\n 3.7011e-04, 2.4151e-04, 3.5867e-04, 3.6172e-04, 1.8963e-05, 1.1769e-04,\n 2.0635e-04, 3.8020e-04, 4.2807e-04, 1.7283e-04, 4.3447e-04, 4.0912e-04,\n 5.0599e-04, 1.1536e-04, 5.6190e-04, 1.3383e-05, 2.8303e-04, 2.1363e-04,\n 6.8672e-04, 3.9874e-04, 3.8517e-04, 2.9272e-08, 2.7493e-04, 3.1867e-04,\n 1.4414e-04, 1.3467e-05, 3.7729e-04, 8.8554e-05, 3.5728e-04, 8.9449e-04,\n 4.0656e-04, 2.4550e-05, 5.0835e-05, 6.4228e-04, 5.6396e-05, 3.2098e-04,\n 3.8251e-04, 8.5874e-05, 1.7040e-07, 3.1945e-04, 2.9853e-04, 5.4198e-04,\n 1.4710e-05, 3.3613e-04, 3.3173e-04, 1.2794e-04, 4.3563e-04, 3.6395e-06,\n 4.1713e-07, 3.5128e-04, 5.4460e-04, 4.9651e-04, 1.5089e-04, 1.1559e-04,\n 4.9165e-04, 4.9700e-05, 3.3630e-05, 1.7088e-04, 2.7242e-04, 2.5457e-05,\n 7.7654e-05, 3.1806e-04, 3.2397e-04, 3.2485e-04, 3.7055e-04, 1.0779e-04,\n 9.9908e-05, 4.3545e-04, 3.8227e-04, 1.4000e-05, 5.3566e-04, 3.7736e-04,\n 3.6501e-04, 4.7370e-04, 1.1287e-09, 4.5773e-05, 6.1692e-05, 2.1292e-08,\n 3.4979e-04, 2.0964e-04, 1.0142e-04, 3.4325e-05, 2.7653e-04, 5.6952e-04,\n 2.8425e-04, 3.3401e-04, 2.0203e-04, 2.0253e-04, 1.6593e-04, 2.1450e-04,\n 6.1481e-04, 2.3255e-04, 3.2677e-04, 1.4508e-04, 2.1946e-04, 8.7082e-06,\n 3.0357e-04, 5.6236e-04, 3.8946e-04, 1.9182e-06, 2.7049e-04, 3.8064e-04,\n 3.1693e-04, 4.0472e-04, 3.9354e-04, 2.2280e-05, 5.4546e-04, 4.6367e-04,\n 3.9682e-04, 2.0302e-04, 3.0279e-04, 3.1841e-04, 2.5411e-04, 6.2497e-05,\n 6.5077e-04, 3.0138e-04, 3.0513e-05, 3.1613e-04, 7.7986e-05, 4.2770e-04,\n 4.9763e-04, 1.8336e-05, 3.3804e-04, 1.1759e-07, 5.6578e-04, 1.3990e-07,\n 1.6763e-05, 7.4505e-05, 7.0571e-05, 8.6862e-05, 5.2583e-04, 2.6228e-04,\n 5.9063e-04, 1.9810e-04, 5.4183e-04, 2.3618e-04, 4.6706e-04, 3.4835e-07,\n 6.2509e-04, 3.3570e-04, 3.1122e-04, 3.6230e-04, 4.1115e-04, 3.1698e-04,\n 4.0838e-04, 1.4070e-04, 5.4830e-06, 2.8906e-05, 2.1062e-04, 4.5388e-04,\n 4.4754e-04, 8.3709e-05, 1.2278e-04, 4.7465e-04, 4.7492e-04, 3.7420e-04,\n 3.2303e-04, 4.8767e-04, 4.1527e-04, 4.4125e-04, 2.6525e-05, 5.1267e-04,\n 1.9342e-04, 3.7850e-04, 1.6718e-04, 2.0448e-04, 8.5436e-05, 1.9403e-04,\n 4.3444e-04, 5.9754e-04, 3.8806e-04, 7.9137e-05, 4.0242e-04, 4.5539e-05,\n 1.7280e-05, 1.2320e-05, 4.1035e-04, 4.7530e-04, 7.5353e-05, 1.6431e-05,\n 1.5094e-04, 2.7329e-04, 3.0586e-04, 1.8898e-05, 3.3463e-04, 5.6591e-04,\n 1.8963e-04, 6.6765e-04, 3.6721e-08, 3.5805e-04, 2.8893e-04, 7.9331e-06,\n 2.2437e-04, 5.9126e-05, 3.3202e-04, 5.7861e-04, 7.6649e-09, 2.3133e-04,\n 3.7428e-04, 4.0493e-04, 2.0453e-04, 6.1575e-04, 4.9658e-04, 4.7897e-04,\n 2.0930e-05, 6.5345e-06, 3.9946e-04, 4.3226e-04, 2.8530e-04, 3.3587e-04,\n 2.9929e-04, 1.4014e-07, 2.0985e-05, 5.0804e-04, 5.8949e-04, 5.2892e-04,\n 3.2512e-04, 1.6550e-05, 6.4113e-04, 6.0734e-06, 2.1985e-04, 3.8531e-05,\n 3.1974e-04, 6.4578e-05, 2.8385e-04, 3.1662e-04, 2.9114e-04, 3.3911e-04,\n 1.2317e-05, 5.3825e-04, 1.8505e-04, 3.9422e-04, 4.1867e-06, 2.4813e-04,\n 9.8120e-05, 6.4542e-05, 3.9994e-04, 1.4179e-04, 6.5725e-04, 1.6070e-04,\n 3.1516e-04, 2.9908e-04, 2.5610e-04, 4.6229e-04, 3.5822e-04, 8.3469e-06,\n 3.0079e-04, 1.2223e-04, 5.5232e-05, 5.5732e-04, 5.7599e-04, 4.3664e-04,\n 7.4226e-05, 2.2818e-04, 3.4307e-04, 3.3908e-04, 2.5616e-04, 1.6313e-07,\n 5.0850e-04, 4.6017e-05, 1.2575e-04, 4.1892e-04, 3.6299e-04, 3.8694e-04,\n 5.3498e-04, 3.3170e-04, 1.6292e-04, 4.0496e-08, 5.3790e-04, 2.0420e-04,\n 3.1416e-04, 2.6129e-04, 6.4147e-05, 2.7407e-04, 2.6269e-04, 2.6862e-04,\n 2.0854e-04, 3.6781e-04, 4.3721e-07, 2.4691e-07, 4.6604e-04, 9.9337e-06,\n 3.7281e-05, 4.0091e-04, 8.1125e-07, 2.5758e-04, 3.9887e-04, 1.6179e-05,\n 2.6248e-04, 2.9560e-04, 4.8587e-04, 4.4239e-04, 3.3841e-04, 1.9262e-04],\n device='cuda:0')" + }, + "4": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.3073e-05, -1.5159e-05, 1.7370e-05, ..., -5.2862e-05,\n 1.0348e-05, -6.8138e-06],\n [-1.7186e-05, 2.7571e-05, 4.8519e-06, ..., 2.6876e-06,\n -2.1980e-05, -1.1533e-05],\n [-7.5210e-06, -1.7147e-05, 3.5318e-06, ..., -7.0512e-05,\n -2.3471e-05, 1.9823e-05],\n ...,\n [-2.0415e-06, -9.7949e-06, 2.2148e-05, ..., 5.4686e-05,\n 6.8761e-06, -2.5319e-05],\n [ 2.2185e-05, 1.7464e-05, -1.4931e-06, ..., -5.8472e-05,\n 4.0500e-05, -4.4397e-05],\n [-1.5582e-05, -2.2352e-05, -2.8526e-05, ..., 2.2091e-05,\n -3.3767e-05, -3.0202e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3531e-09, 3.2064e-09, 1.4878e-08, ..., 5.1072e-08, 2.8909e-08,\n 1.1841e-08],\n [1.7752e-09, 4.4280e-09, 2.2355e-08, ..., 7.1289e-08, 5.8437e-08,\n 1.6403e-08],\n [4.1926e-09, 5.1599e-09, 2.7594e-08, ..., 1.1226e-07, 6.1464e-08,\n 1.8121e-08],\n ...,\n [1.7911e-09, 6.8512e-09, 2.6307e-08, ..., 7.0327e-08, 4.0744e-08,\n 1.8873e-08],\n [7.4002e-09, 5.6401e-09, 2.7232e-08, ..., 9.7110e-08, 3.5312e-08,\n 2.1041e-08],\n [3.4450e-09, 4.9187e-09, 2.3970e-08, ..., 5.7287e-08, 3.4531e-08,\n 1.7985e-08]], device='cuda:0')" + }, + "5": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.6560e-06, 1.4190e-06, 5.6052e-45, ..., -5.6427e-06,\n -6.1125e-06, 5.6052e-45],\n [-6.6330e-07, 2.0823e-07, -5.6052e-45, ..., 5.1263e-06,\n -5.0665e-06, -5.6052e-45],\n [-1.6435e-05, -9.7737e-06, 5.6052e-45, ..., 3.1497e-06,\n 1.8028e-05, -5.6052e-45],\n ...,\n [ 3.8464e-06, -1.6066e-06, 5.6052e-45, ..., -3.7617e-07,\n -5.9910e-06, -5.6052e-45],\n [ 1.6530e-06, 1.6435e-05, -5.6052e-45, ..., -2.5091e-06,\n 5.3747e-06, 5.6052e-45],\n [-1.4230e-06, -1.0071e-05, 5.6052e-45, ..., 6.7412e-06,\n 5.0700e-05, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.5469e-09, 5.5843e-10, 5.6998e-14, ..., 9.6906e-10, 2.1604e-09,\n 1.0646e-14],\n [2.0113e-09, 3.3223e-09, 6.2024e-11, ..., 2.3720e-09, 7.3774e-10,\n 4.4304e-10],\n [3.5116e-09, 3.4628e-09, 1.9223e-11, ..., 5.9705e-09, 5.7780e-09,\n 5.1205e-11],\n ...,\n [1.4332e-09, 4.1079e-09, 1.8643e-11, ..., 3.8868e-09, 2.0786e-09,\n 4.4161e-12],\n [1.8874e-09, 1.2690e-09, 1.4147e-11, ..., 3.5837e-09, 1.9831e-09,\n 6.5358e-12],\n [3.4413e-09, 4.5935e-09, 3.2231e-12, ..., 3.3446e-09, 2.7845e-09,\n 5.9420e-11]], device='cuda:0')" + }, + "6": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 0.0013, 0.0005, -0.0008, ..., 0.0005, 0.0002, 0.0016],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.0022e-05, 3.1277e-05, 5.6109e-05, ..., 4.3793e-05, 4.5151e-05,\n 4.6691e-05], device='cuda:0')" + }, + "7": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.1672e-06, 7.2977e-06, -3.9535e-07, ..., 3.6100e-06,\n 5.5741e-06, -5.2066e-06],\n [ 1.8975e-05, 2.7750e-06, 1.6962e-05, ..., 9.3662e-06,\n 3.4394e-06, 2.1873e-05],\n [-1.9219e-06, -1.9377e-05, -4.9753e-05, ..., -2.2481e-06,\n -4.5406e-06, -2.1614e-05],\n ...,\n [ 9.3896e-06, -4.4725e-05, 7.3406e-06, ..., -4.9999e-06,\n -1.5420e-06, -1.5970e-05],\n [-2.2596e-05, -2.5074e-05, -6.4039e-06, ..., 3.0068e-06,\n -3.6128e-06, -1.6375e-05],\n [ 2.9142e-05, -5.6812e-06, 1.0102e-05, ..., 2.1704e-06,\n -5.4963e-06, -6.4921e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.1024e-09, 2.3446e-09, 2.7110e-09, ..., 1.6306e-09, 2.5175e-09,\n 2.8967e-09],\n [2.9976e-09, 4.2550e-09, 5.4079e-09, ..., 3.8396e-09, 4.7828e-09,\n 4.7081e-09],\n [2.4268e-09, 4.0720e-09, 5.5368e-09, ..., 2.9404e-09, 5.0089e-09,\n 6.2223e-09],\n ...,\n [2.6656e-09, 5.3142e-09, 5.3576e-09, ..., 3.2378e-09, 4.6503e-09,\n 5.8959e-09],\n [3.3780e-09, 4.5906e-09, 4.6166e-09, ..., 3.6453e-09, 5.3069e-09,\n 4.6821e-09],\n [3.3600e-09, 5.0319e-09, 5.6386e-09, ..., 3.6028e-09, 5.4241e-09,\n 5.3720e-09]], device='cuda:0')" + }, + "8": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.6052e-45, 2.0869e-20, 5.6052e-45, ..., -5.6052e-45,\n 1.5994e-19, 5.6052e-45],\n [ 6.0104e-06, -1.2322e-06, -5.6052e-45, ..., 5.7030e-06,\n 2.6388e-05, 5.6052e-45],\n [ 1.7568e-06, 3.9494e-06, -5.6052e-45, ..., 4.2572e-07,\n 1.2515e-05, 5.6052e-45],\n ...,\n [-1.2234e-06, 3.1359e-06, -5.6052e-45, ..., 3.7677e-06,\n 5.6791e-06, 5.6052e-45],\n [-1.1802e-06, 3.9850e-07, -5.6052e-45, ..., -6.1178e-06,\n 1.6768e-05, 5.6052e-45],\n [ 2.3519e-10, -5.6930e-08, 5.6052e-45, ..., -1.3086e-06,\n -2.1160e-08, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.5211e-11, 8.1043e-13, 1.4688e-11, ..., 1.0633e-12, 2.2666e-13,\n 2.2664e-10],\n [1.1588e-08, 7.5112e-09, 1.2669e-12, ..., 1.1574e-08, 1.3307e-08,\n 1.8355e-11],\n [4.6111e-09, 1.0201e-08, 4.5228e-12, ..., 1.6990e-08, 1.9263e-08,\n 3.6957e-12],\n ...,\n [8.2230e-09, 3.6288e-09, 1.7328e-13, ..., 1.7288e-08, 1.3817e-08,\n 7.7673e-12],\n [5.4678e-09, 4.9387e-09, 1.2076e-11, ..., 1.5277e-08, 1.3423e-08,\n 1.4246e-11],\n [4.3955e-12, 2.7582e-10, 1.2147e-12, ..., 2.5982e-09, 4.7957e-12,\n 7.5646e-11]], device='cuda:0')" + }, + "9": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 5.9960e-11, 1.6456e-03, 2.9527e-04, ..., -2.7276e-05,\n 2.5739e-04, 2.4684e-04], device='cuda:0')", + "exp_avg_sq": "tensor([1.7745e-07, 2.6573e-04, 3.9830e-04, ..., 4.1268e-04, 2.7468e-04,\n 1.0156e-05], device='cuda:0')" + }, + "10": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-1.3421e-14, -9.3161e-07, -1.5511e-06, ..., 2.3660e-06,\n 1.2268e-05, 1.3369e-06],\n [ 1.0584e-14, 9.7407e-06, 1.4394e-05, ..., 1.7064e-06,\n -2.2420e-05, 1.1143e-06],\n [-5.8117e-16, -1.5219e-05, 1.0313e-06, ..., -4.6790e-06,\n -1.0698e-05, -2.3850e-06],\n ...,\n [-2.4283e-14, -4.0979e-06, -1.7563e-06, ..., 7.1534e-08,\n -1.5975e-05, -1.4058e-06],\n [ 6.9043e-15, -1.0240e-05, -3.5480e-05, ..., 8.3860e-06,\n 6.5756e-06, -2.9546e-06],\n [-1.7200e-15, -3.1496e-05, 6.0733e-06, ..., -3.5074e-06,\n -2.1187e-05, -2.4709e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3467e-13, 7.8071e-09, 1.6498e-08, ..., 1.1281e-08, 7.9192e-09,\n 5.4617e-11],\n [5.6968e-12, 2.0681e-08, 2.5243e-08, ..., 2.3040e-08, 1.3985e-08,\n 5.7467e-11],\n [2.6576e-12, 1.4105e-08, 1.8451e-08, ..., 2.1005e-08, 1.4557e-08,\n 6.8354e-11],\n ...,\n [7.7940e-12, 1.3184e-08, 2.4794e-08, ..., 1.6709e-08, 1.2125e-08,\n 7.5400e-11],\n [1.3680e-11, 1.1839e-08, 1.5446e-08, ..., 2.1700e-08, 1.5078e-08,\n 1.1377e-10],\n [1.3510e-11, 1.7260e-08, 2.0059e-08, ..., 1.7792e-08, 1.2607e-08,\n 6.9416e-11]], device='cuda:0')" + }, + "11": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 6.6213e-06, -3.2795e-07, 5.6052e-45, ..., 9.4162e-07,\n -1.5536e-05, -5.6052e-45],\n [-4.1859e-06, 9.2341e-06, -5.6052e-45, ..., 6.6641e-06,\n 3.4662e-07, -5.6052e-45],\n [-8.0553e-06, 4.2282e-06, -5.6052e-45, ..., -2.2435e-06,\n 9.7187e-06, -5.6052e-45],\n ...,\n [ 1.9016e-29, 5.6052e-45, 5.6052e-45, ..., 1.3235e-17,\n -5.6052e-45, 5.6052e-45],\n [ 3.3999e-05, -7.8092e-06, 5.6052e-45, ..., -1.0152e-05,\n 4.5596e-06, -5.6052e-45],\n [-2.6870e-06, 4.4262e-06, 5.6052e-45, ..., -3.1033e-06,\n -1.3463e-05, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[8.7760e-10, 1.4584e-10, 4.8139e-12, ..., 1.4354e-09, 1.8902e-09,\n 2.9478e-13],\n [7.1889e-10, 1.5839e-09, 1.8807e-13, ..., 1.7227e-09, 8.7139e-10,\n 2.3278e-11],\n [2.1661e-09, 2.8272e-09, 5.6044e-12, ..., 2.0084e-09, 9.5032e-10,\n 2.3609e-10],\n ...,\n [3.5741e-11, 8.8220e-13, 2.9899e-12, ..., 7.2797e-15, 2.4456e-13,\n 1.5923e-10],\n [2.9899e-09, 1.1455e-08, 3.6391e-12, ..., 8.6412e-09, 2.6878e-10,\n 1.1357e-11],\n [3.8894e-09, 2.3272e-09, 1.9249e-13, ..., 1.3195e-09, 2.4564e-09,\n 1.2228e-11]], device='cuda:0')" + }, + "12": { + "step": "tensor(1252.)", + "exp_avg": "tensor([7.2584e-05, 2.7397e-04, 9.9030e-05, ..., 7.3802e-16, 4.0785e-04,\n 5.9190e-05], device='cuda:0')", + "exp_avg_sq": "tensor([2.7723e-05, 2.0981e-05, 1.5218e-05, ..., 2.3416e-07, 8.3553e-06,\n 2.3908e-05], device='cuda:0')" + }, + "13": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-2.8632e-06, 2.0202e-06, -1.8505e-06, ..., 2.2188e-18,\n -1.6002e-06, -3.9616e-06],\n [-4.3427e-06, 3.3626e-07, 1.6469e-06, ..., 4.3351e-19,\n -5.8539e-06, 1.1954e-07],\n [ 1.8464e-08, -1.0925e-06, 3.5800e-06, ..., 8.2445e-19,\n 7.4627e-07, -1.5925e-06],\n ...,\n [-4.0516e-06, -1.1619e-05, 6.8448e-06, ..., 2.5324e-18,\n 4.2783e-06, 2.2592e-06],\n [-4.3855e-06, 1.3411e-05, 1.4463e-05, ..., 9.8912e-19,\n -1.1521e-06, 5.6119e-06],\n [ 2.4618e-06, -4.7924e-06, 5.2861e-06, ..., 2.5677e-18,\n 1.4157e-06, -2.8334e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.8733e-10, 5.1638e-10, 3.0455e-10, ..., 1.6012e-11, 1.6676e-10,\n 3.4313e-10],\n [3.5922e-10, 7.7585e-10, 6.3230e-10, ..., 1.8755e-11, 2.2945e-10,\n 4.5819e-10],\n [4.8408e-10, 7.0879e-10, 6.5761e-10, ..., 1.9174e-11, 2.5860e-10,\n 8.4777e-10],\n ...,\n [6.1272e-10, 1.1979e-09, 5.0757e-10, ..., 4.0283e-13, 1.9568e-10,\n 5.4206e-10],\n [4.7023e-10, 7.1243e-10, 6.5220e-10, ..., 4.7369e-11, 2.3447e-10,\n 7.4886e-10],\n [5.7807e-10, 9.2635e-10, 8.3888e-10, ..., 1.8072e-11, 2.8465e-10,\n 7.8391e-10]], device='cuda:0')" + }, + "14": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-7.6659e-26, 3.5712e-26, 5.6052e-45, ..., 1.8236e-12,\n -1.2910e-25, 5.6052e-45],\n [-1.9373e-06, -3.9706e-06, -5.6052e-45, ..., 4.6498e-05,\n 2.0514e-05, -5.6052e-45],\n [ 3.1648e-05, -1.2480e-04, 5.6052e-45, ..., -4.6819e-06,\n -1.2046e-05, 5.6052e-45],\n ...,\n [-4.8492e-05, 1.4720e-05, -5.6052e-45, ..., -5.7093e-05,\n 1.8878e-07, -5.6052e-45],\n [-1.9728e-05, 1.9350e-06, 5.6052e-45, ..., 1.3352e-06,\n -3.5115e-05, 5.6052e-45],\n [-4.2228e-05, -6.2994e-05, 5.6052e-45, ..., 5.1379e-06,\n -6.7046e-06, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.1432e-11, 8.0671e-13, 1.0777e-12, ..., 8.1770e-12, 1.1459e-12,\n 8.8919e-12],\n [4.6180e-08, 1.2855e-08, 2.6597e-14, ..., 5.3700e-08, 1.2771e-07,\n 1.2124e-12],\n [2.5421e-08, 2.2602e-08, 3.6156e-14, ..., 5.6282e-08, 3.6397e-08,\n 1.1308e-11],\n ...,\n [3.2328e-08, 2.6572e-08, 1.9920e-12, ..., 2.9913e-08, 1.3796e-07,\n 4.0315e-12],\n [1.0849e-08, 2.5898e-08, 1.9059e-13, ..., 2.3549e-08, 9.3493e-09,\n 4.4472e-13],\n [1.1001e-08, 3.7065e-08, 4.5399e-13, ..., 2.0455e-08, 2.7765e-08,\n 7.8892e-12]], device='cuda:0')" + }, + "15": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 9.0159e-09, 1.1229e-03, 1.8014e-03, ..., -6.2551e-03,\n 7.8081e-04, 6.9722e-04], device='cuda:0')", + "exp_avg_sq": "tensor([1.1850e-07, 4.3919e-04, 3.4083e-04, ..., 4.9122e-04, 3.8708e-04,\n 4.4057e-04], device='cuda:0')" + }, + "16": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-6.4129e-11, -5.8002e-06, -2.1526e-05, ..., 1.1727e-05,\n 8.3311e-07, 7.0070e-06],\n [-5.4931e-11, 1.1517e-05, 9.8675e-07, ..., -4.6584e-05,\n -3.9126e-05, 3.6394e-07],\n [-5.9968e-11, 7.8650e-06, 1.9704e-05, ..., -6.8655e-06,\n 2.6962e-06, 1.3676e-05],\n ...,\n [ 4.4871e-11, -1.1433e-05, 8.8597e-06, ..., -9.1967e-06,\n -1.6298e-05, 9.4056e-07],\n [-1.7269e-10, -1.3450e-05, -2.6279e-05, ..., 1.3633e-05,\n -4.7477e-06, -8.4443e-06],\n [-2.8507e-11, -2.0172e-05, -3.1173e-06, ..., -6.0291e-06,\n -1.9764e-06, -6.9866e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7323e-10, 5.2698e-09, 4.1363e-09, ..., 3.5805e-09, 2.6277e-09,\n 3.0772e-09],\n [1.8467e-10, 6.4333e-09, 5.7830e-09, ..., 7.1143e-09, 4.8912e-09,\n 8.1809e-09],\n [1.0490e-10, 1.0148e-08, 5.6220e-09, ..., 6.5901e-09, 5.4333e-09,\n 7.8561e-09],\n ...,\n [1.6738e-10, 8.5928e-09, 6.8384e-09, ..., 6.2710e-09, 5.1333e-09,\n 9.7322e-09],\n [5.6434e-10, 1.7796e-08, 6.7818e-09, ..., 6.2551e-09, 4.9200e-09,\n 1.1089e-08],\n [1.7990e-10, 1.2667e-08, 5.9724e-09, ..., 6.8928e-09, 5.5055e-09,\n 6.1507e-09]], device='cuda:0')" + }, + "17": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.5441e-07, 1.0542e-05, 5.6052e-45, ..., 5.7532e-06,\n -1.5153e-05, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n ...,\n [ 1.0630e-07, 1.3339e-07, 5.6052e-45, ..., 3.0354e-06,\n 5.1922e-06, 5.6052e-45],\n [-2.5619e-08, 7.8163e-08, -5.6052e-45, ..., 6.5206e-07,\n 6.1654e-07, -5.6052e-45],\n [-5.3497e-07, 1.0002e-05, -5.6052e-45, ..., 1.6710e-06,\n 9.5171e-06, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.1304e-09, 2.4858e-09, 3.1587e-12, ..., 2.3953e-09, 4.3429e-09,\n 2.3251e-11],\n [1.9858e-13, 2.5914e-14, 3.6196e-13, ..., 3.5015e-13, 1.5648e-15,\n 3.4683e-12],\n [5.4457e-11, 4.1213e-16, 1.7431e-11, ..., 4.5127e-12, 3.9030e-17,\n 9.4992e-11],\n ...,\n [5.5297e-11, 1.3433e-10, 4.1577e-12, ..., 1.2377e-09, 1.8984e-10,\n 2.9399e-11],\n [2.9196e-11, 1.5456e-11, 7.8489e-13, ..., 1.9851e-10, 1.0531e-10,\n 1.4844e-11],\n [2.6734e-10, 4.6963e-10, 1.9464e-13, ..., 1.7416e-09, 2.6676e-10,\n 4.8682e-12]], device='cuda:0')" + }, + "18": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-3.9034e-04, 5.6052e-45, 1.2100e-18, ..., -2.3966e-04,\n -8.9872e-04, 5.9991e-04], device='cuda:0')", + "exp_avg_sq": "tensor([6.1793e-05, 1.0890e-08, 1.4690e-07, ..., 7.6825e-05, 5.0104e-05,\n 3.9163e-05], device='cuda:0')" + }, + "19": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-9.2946e-06, 5.6052e-45, -2.1733e-21, ..., 2.3871e-07,\n 4.3282e-06, -3.5452e-06],\n [ 6.5502e-07, -5.6052e-45, -6.5234e-21, ..., 9.5066e-08,\n 4.3630e-06, 2.6021e-06],\n [ 4.7432e-06, 5.6052e-45, -1.1182e-20, ..., 3.5316e-07,\n -3.9039e-06, 3.7818e-06],\n ...,\n [-1.1370e-06, 5.6052e-45, -1.3608e-20, ..., 1.1156e-06,\n -5.9590e-07, 9.7333e-07],\n [ 3.5577e-06, -5.6052e-45, 3.8891e-21, ..., 4.5445e-07,\n 1.6080e-06, 7.6327e-06],\n [ 2.0133e-06, -5.6052e-45, 1.0891e-20, ..., -3.3061e-06,\n 1.2826e-05, 7.3350e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.0966e-10, 4.2241e-14, 2.2743e-13, ..., 2.3046e-10, 1.1113e-10,\n 2.7489e-10],\n [6.9676e-10, 6.7421e-15, 1.1556e-13, ..., 5.0720e-10, 7.5217e-10,\n 2.5459e-10],\n [4.6137e-10, 2.7551e-14, 5.9818e-12, ..., 9.3089e-10, 4.5391e-10,\n 3.0583e-10],\n ...,\n [3.4071e-10, 7.7055e-14, 1.9046e-13, ..., 4.9141e-10, 4.4609e-10,\n 4.9023e-10],\n [7.5494e-10, 3.0604e-16, 1.4392e-12, ..., 9.3960e-10, 2.7760e-10,\n 2.2888e-10],\n [5.9897e-10, 2.3736e-16, 2.6719e-13, ..., 5.7415e-10, 1.2134e-09,\n 4.3760e-10]], device='cuda:0')" + }, + "20": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-2.4576e-07, 6.2080e-06, -5.6052e-45, ..., 1.3158e-06,\n -5.2843e-07, -5.6052e-45],\n [-1.8715e-08, -1.1064e-07, 5.6052e-45, ..., 6.8184e-08,\n 4.3245e-07, 5.6052e-45],\n [-2.3099e-09, -3.4216e-19, 5.6052e-45, ..., -2.7125e-09,\n -1.7051e-09, -5.6052e-45],\n ...,\n [ 2.7452e-06, -1.9578e-06, -5.6052e-45, ..., 6.6394e-06,\n -1.4816e-05, -5.6052e-45],\n [-2.3086e-06, 2.2816e-06, -5.6052e-45, ..., 4.5497e-07,\n 1.5765e-06, 5.6052e-45],\n [ 5.6052e-45, -4.7203e-26, 5.6052e-45, ..., 7.6457e-22,\n 1.6340e-27, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.8595e-10, 3.1539e-09, 2.1554e-13, ..., 1.2735e-09, 9.4787e-10,\n 3.2032e-13],\n [1.0866e-10, 3.2451e-12, 6.6958e-12, ..., 3.3911e-11, 1.7035e-10,\n 1.7416e-11],\n [2.5588e-11, 1.0837e-12, 4.4739e-13, ..., 1.1843e-11, 7.2697e-12,\n 3.4028e-11],\n ...,\n [8.0069e-10, 2.9750e-09, 3.3405e-12, ..., 8.9813e-10, 1.2842e-09,\n 7.3276e-11],\n [5.5957e-10, 4.5423e-10, 7.3053e-12, ..., 1.3726e-09, 7.5665e-10,\n 2.9305e-11],\n [4.3488e-11, 1.3656e-14, 1.0543e-11, ..., 2.0256e-12, 2.6338e-14,\n 6.0910e-12]], device='cuda:0')" + }, + "21": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 2.8944e-04, -3.7704e-04, 1.4071e-06, ..., 3.3318e-04,\n 3.0178e-04, 4.6544e-19], device='cuda:0')", + "exp_avg_sq": "tensor([1.5167e-05, 1.2036e-05, 1.3312e-07, ..., 8.2531e-06, 1.5794e-05,\n 3.7906e-08], device='cuda:0')" + }, + "22": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 1.4123e-07, 4.5098e-07, -4.6478e-09, ..., -1.1021e-06,\n 2.7191e-07, -2.4202e-22],\n [ 4.5152e-07, -7.0374e-07, -2.5558e-09, ..., 8.6171e-07,\n -3.5182e-07, -8.9231e-22],\n [ 1.8528e-06, -6.1058e-07, -1.5119e-08, ..., -4.4964e-06,\n -3.4436e-06, 4.7549e-22],\n ...,\n [-5.0061e-07, -3.3780e-07, 1.0466e-08, ..., -3.5104e-06,\n -2.4252e-06, 8.3319e-22],\n [-1.1725e-07, 1.3464e-07, 7.1544e-09, ..., 2.7005e-06,\n -6.7828e-07, 2.7287e-21],\n [ 2.8806e-07, 7.0134e-07, 9.1836e-09, ..., -7.0473e-07,\n 1.1137e-06, -2.1240e-21]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.8183e-11, 1.4972e-11, 1.8150e-11, ..., 8.0029e-11, 7.1775e-11,\n 6.2146e-13],\n [1.3844e-10, 4.9363e-11, 5.6433e-12, ..., 4.7497e-11, 1.3941e-10,\n 6.0144e-13],\n [1.4843e-10, 4.0618e-11, 7.5600e-12, ..., 4.3009e-11, 1.3116e-10,\n 2.5523e-12],\n ...,\n [7.6370e-11, 2.7970e-11, 3.2560e-12, ..., 4.1760e-11, 1.3649e-10,\n 9.7815e-13],\n [1.5617e-10, 2.8068e-11, 4.8958e-12, ..., 5.5827e-11, 1.6919e-10,\n 1.4927e-13],\n [9.0328e-11, 2.6521e-11, 1.3065e-11, ..., 8.5808e-11, 1.5359e-10,\n 1.4745e-12]], device='cuda:0')" + }, + "23": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 8.3853e-38, 6.8410e-19, 5.6052e-45, ..., 5.6052e-45,\n 2.0687e-38, 5.6052e-45],\n [ 4.5093e-32, 1.8352e-29, -5.6052e-45, ..., -1.0451e-29,\n 5.3404e-30, 5.6052e-45],\n [-1.6074e-06, -7.1780e-07, -5.6052e-45, ..., -7.6210e-08,\n -6.9990e-06, 5.6052e-45],\n ...,\n [ 9.1965e-08, 7.7571e-08, 5.6052e-45, ..., -1.2016e-07,\n -7.3938e-06, 5.6052e-45],\n [-3.1388e-08, -5.6622e-07, -5.6052e-45, ..., 9.4448e-06,\n -7.1260e-06, -5.6052e-45],\n [-2.0214e-06, 5.5312e-06, 5.6052e-45, ..., 1.4612e-06,\n 1.9302e-05, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.2290e-12, 3.4537e-14, 1.4568e-12, ..., 9.3933e-12, 4.4817e-13,\n 4.0472e-11],\n [3.4973e-11, 2.9460e-13, 2.0194e-12, ..., 1.0674e-12, 6.7694e-15,\n 1.2358e-11],\n [1.5575e-09, 1.2480e-09, 2.5419e-12, ..., 8.5893e-10, 1.1694e-09,\n 6.8070e-12],\n ...,\n [2.8394e-09, 1.2162e-10, 2.5632e-12, ..., 1.4100e-09, 8.9839e-10,\n 2.3707e-12],\n [8.4949e-11, 8.4284e-10, 9.8242e-13, ..., 2.7051e-09, 1.0559e-09,\n 1.6330e-11],\n [1.8270e-09, 7.2251e-10, 1.2474e-12, ..., 3.8758e-09, 4.1247e-09,\n 2.1698e-11]], device='cuda:0')" + }, + "24": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 1.4612e-16, 1.4556e-26, 2.1739e-04, ..., -4.1949e-04,\n -6.4733e-05, 2.1239e-03], device='cuda:0')", + "exp_avg_sq": "tensor([5.7599e-08, 7.7426e-09, 1.3828e-05, ..., 1.7633e-05, 1.5016e-05,\n 4.7332e-05], device='cuda:0')" + }, + "25": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.9231e-19, 5.7139e-31, -1.8612e-07, ..., 1.7457e-07,\n 2.9930e-07, -3.8831e-07],\n [ 1.3586e-18, -5.0848e-29, -2.2667e-07, ..., 1.2408e-06,\n -1.2990e-06, 2.0436e-06],\n [-5.4389e-19, -1.2886e-28, -8.5975e-07, ..., 5.7523e-07,\n 3.0375e-06, 2.1489e-07],\n ...,\n [ 2.2288e-19, 4.6604e-29, -5.0366e-07, ..., -2.3322e-07,\n 3.1312e-07, -2.2734e-07],\n [-2.5571e-19, -1.9165e-29, -3.0710e-07, ..., -5.0593e-07,\n -9.8576e-07, -3.7464e-08],\n [-1.0379e-18, -4.7826e-29, -1.5544e-06, ..., 1.2024e-06,\n -3.0276e-07, -2.4213e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7124e-15, 1.8045e-12, 3.7880e-11, ..., 5.0256e-11, 5.2542e-11,\n 4.3494e-11],\n [1.0623e-11, 1.4964e-11, 1.1510e-10, ..., 6.1617e-11, 6.5964e-11,\n 8.2406e-11],\n [8.1316e-13, 2.3055e-12, 2.7175e-10, ..., 8.0111e-11, 6.2162e-11,\n 1.6026e-10],\n ...,\n [8.4432e-13, 1.7031e-11, 8.5927e-11, ..., 9.2773e-11, 1.1189e-10,\n 1.4866e-10],\n [1.3109e-17, 1.2611e-11, 3.8145e-10, ..., 1.8282e-10, 1.1996e-10,\n 1.7188e-10],\n [6.9688e-12, 1.4160e-11, 3.2440e-10, ..., 1.0703e-10, 9.4980e-11,\n 1.5254e-10]], device='cuda:0')" + }, + "26": { + "step": "tensor(1252.)", + "exp_avg": "tensor([1.4280e-10], device='cuda:0')", + "exp_avg_sq": "tensor([1.5982e-05], device='cuda:0')" + }, + "27": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-2.5338e-13, 1.3276e-12, -1.0742e-12], device='cuda:0')", + "exp_avg_sq": "tensor([7.2189e-08, 2.2982e-07, 6.2485e-08], device='cuda:0')" + }, + "28": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 2.5278e-36, 5.6052e-45, -5.6052e-45, -1.2928e-12, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45], device='cuda:0')", + "exp_avg_sq": "tensor([1.0545e-05, 4.6682e-07, 8.2353e-06, 4.5063e-07, 4.7069e-07, 9.1760e-08,\n 3.6277e-07, 3.7775e-07], device='cuda:0')" + }, + "30": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-9.0921e-19, 5.8820e-18, -5.6052e-45, ..., -1.5531e-16,\n -5.7569e-16, 5.6052e-45],\n [-5.5844e-17, -5.3157e-17, 5.6052e-45, ..., -3.1521e-16,\n -1.4523e-14, -5.6052e-45],\n [ 1.1146e-17, -3.3739e-18, -5.6052e-45, ..., 1.4284e-16,\n 1.8936e-15, 5.6052e-45],\n ...,\n [ 8.6086e-19, -1.2911e-17, -5.6052e-45, ..., -1.0834e-15,\n -6.9578e-16, 5.6052e-45],\n [-6.2218e-17, 4.8925e-17, 5.6052e-45, ..., -5.8427e-16,\n -5.5040e-15, -5.6052e-45],\n [ 4.0211e-18, -5.9720e-18, -5.6052e-45, ..., 4.6844e-16,\n 3.6466e-16, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7017e-12, 1.6420e-12, 3.8961e-16, ..., 2.8398e-12, 4.3632e-13,\n 2.6412e-14],\n [1.1144e-12, 7.3989e-13, 1.3874e-17, ..., 5.8014e-13, 3.5541e-13,\n 1.0673e-14],\n [9.3243e-13, 3.9423e-13, 2.6250e-15, ..., 1.6001e-12, 2.6705e-14,\n 4.1383e-13],\n ...,\n [7.1141e-12, 3.2832e-12, 4.9113e-16, ..., 2.8149e-12, 7.7069e-13,\n 1.3160e-15],\n [1.9710e-12, 1.6188e-12, 1.3121e-14, ..., 3.7612e-12, 6.7737e-13,\n 8.2345e-14],\n [3.1098e-12, 6.0495e-12, 8.2815e-17, ..., 4.8038e-12, 1.4082e-12,\n 2.2195e-15]], device='cuda:0')" + }, + "31": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.4518e-13, -1.2292e-12, 1.9316e-13, 1.0537e-13, -6.3684e-14,\n 4.9823e-14, -3.3152e-13, -3.9821e-13, -5.4967e-14, 1.9034e-13,\n 2.8786e-13, 1.3577e-13, 1.3915e-13, -4.8336e-14, -8.4801e-14,\n 1.7734e-13, -1.7469e-12, -2.1501e-14, 3.6072e-14, 1.4990e-13,\n 3.3858e-13, -1.5429e-12, 5.0585e-14, 1.0709e-12, 8.1644e-14,\n -2.2022e-13, -2.2311e-13, 7.9754e-13, -5.9426e-14, -4.1043e-13,\n 5.0957e-13, -2.0685e-13, 2.2999e-13, -3.4944e-13, 4.9646e-13,\n 1.5829e-13, 3.2159e-13, 5.3132e-13, 2.3304e-13, 4.4745e-13,\n 7.5270e-13, 2.8274e-13, 2.0777e-13, -1.3168e-13, -1.1079e-12,\n -1.5271e-14, -1.2774e-12, 2.5212e-14, -9.0562e-14, -3.3196e-13,\n 3.3507e-13, 3.5958e-13, 8.9449e-13, 1.0976e-13, -1.4533e-13,\n 7.2612e-14, -2.4473e-13, -3.0100e-14, 1.4352e-13, -1.0271e-14,\n -7.6865e-13, -2.8186e-14, 2.5748e-15, 1.7776e-13, 6.9257e-14,\n -1.1746e-13, 1.3684e-14, 7.7492e-13, 1.0529e-13, 2.8970e-13,\n -1.6122e-13, 7.2166e-13, -2.1082e-13, 1.5313e-12, 9.6706e-14,\n 3.8284e-13, -1.2169e-12, 1.1277e-14, -6.4321e-13, 9.0215e-13,\n 1.9721e-13, -5.6084e-13, 7.6455e-13, -1.3810e-13, 7.6188e-13,\n 2.3846e-14, -1.0598e-13, -2.1148e-13, -3.8426e-14, -1.3038e-13,\n 1.3072e-13, -1.1603e-13, -2.1686e-13, -3.8352e-13, -1.1441e-13,\n -6.3762e-13, -6.5874e-13, -4.0038e-13, 2.1932e-13, -2.5329e-13,\n 5.4418e-14, 4.8520e-13, -5.1183e-14, 4.0364e-14, -6.6165e-14,\n -5.4597e-13, -3.6938e-15, 4.4189e-13, 1.3997e-13, -6.3735e-13,\n -2.6030e-13, -3.4024e-14, 5.7789e-14, 1.3091e-13, 5.3143e-15,\n 1.3165e-13, -1.4087e-13, 8.0515e-13, -5.1656e-13, 1.7392e-13,\n 3.0432e-14, 6.1859e-14, -5.3596e-13, 5.2765e-14, -1.1131e-12,\n -2.8304e-13, -2.4547e-13, -8.2094e-14, -3.6110e-14, -2.3517e-13,\n 1.2133e-13, 1.1374e-13, 4.2185e-13, -2.3348e-14, 5.6861e-14,\n -7.7138e-13, 5.3125e-13, -4.3987e-15, -4.9299e-13, 4.5094e-13,\n 7.6773e-14, -8.8795e-13, 1.3292e-12, 1.2195e-12, -8.1572e-14,\n 3.8383e-13, -1.9861e-13, -5.2860e-15, 1.3945e-13, 3.7346e-13,\n 3.3721e-13, -5.8061e-14, -2.1061e-13, -4.3097e-13, 2.7976e-13,\n 1.6696e-13, 6.5172e-13, -3.0055e-13, -1.3196e-13, -1.7810e-13,\n -9.4324e-14, 7.2698e-13, -1.0584e-13, 7.5399e-13, -2.9323e-13,\n -1.2760e-13, -1.2347e-14, -1.0962e-13, -5.8095e-13, 1.5815e-14,\n 5.0720e-13, 5.4146e-13, 2.1818e-13, -4.5266e-13, -6.6590e-14,\n 7.4580e-14, -3.4803e-14, 2.0988e-13, 2.2926e-13, -1.4852e-13,\n -6.1144e-13, 3.6785e-13, 2.8925e-13, 7.1080e-16, -1.1522e-12,\n 2.2919e-13, -1.1534e-13, 2.3289e-13, -1.0540e-12, 7.0142e-13,\n 2.4319e-13, -1.2448e-13, 3.4171e-14, 2.9757e-13, 4.6596e-13,\n 2.0712e-13, -2.2569e-13, -1.6212e-13, -1.4730e-13, 1.0693e-13,\n -1.9662e-13, -2.9700e-14, 3.4011e-13, -8.9628e-13, 4.1917e-13,\n 2.3534e-13, -1.1089e-13, -5.2459e-13, -4.5225e-14, 1.7130e-13,\n 6.9635e-14, -5.6009e-14, 1.7344e-13, 1.8121e-13, 7.5339e-13,\n 1.4158e-13, -2.4887e-14, 3.8946e-13, 1.3030e-13, -3.4639e-13,\n 1.5478e-13, 8.8162e-13, -1.6897e-13, 3.5511e-14, 7.3768e-13,\n -2.7172e-13, -3.7275e-13, -4.8048e-13, -6.8142e-13, 1.1408e-13,\n 6.4539e-14, -1.6323e-13, 6.3951e-13, -7.8213e-13, 7.1264e-13,\n 2.2310e-13, -1.4228e-13, 5.5681e-14, 1.0942e-14, 1.0279e-12,\n 6.1589e-13, 6.2259e-13, 1.5578e-14, -4.3296e-14, 3.4635e-13,\n 1.8143e-13, 9.4718e-14, -1.1598e-13, -3.2665e-14, -1.4501e-13,\n -3.6238e-13, 9.9652e-13, 2.3670e-13, -1.0401e-13, 1.0416e-13,\n -4.6048e-14, -6.6719e-13, 2.1590e-13, -8.9895e-14, -3.4580e-13,\n -4.6940e-15, 4.9423e-14, 2.4470e-13, 1.2714e-13, 6.7512e-13,\n -1.5763e-13, 1.8486e-13, -1.1304e-13, 1.1430e-13, -1.8038e-13,\n 5.8767e-13, -1.0487e-13, 1.1725e-13, -7.6342e-13, 3.7367e-13,\n -1.1891e-14, -3.0443e-13, 3.5254e-13, 7.1659e-13, 1.1747e-13,\n -4.3897e-13, 1.1979e-13, 2.0841e-13, -3.2269e-13, 8.5496e-13,\n -2.0278e-12, 2.0391e-13, -1.8756e-13, -4.5244e-13, 8.0229e-13,\n -5.8499e-13, -9.6215e-14, -1.0872e-13, 1.2789e-12, 2.0246e-13,\n 8.8988e-14, -1.1242e-12, -1.9064e-13, 1.7292e-13, 4.1055e-13,\n -3.1840e-13, 1.8010e-13, -9.7065e-14, 6.7979e-13, 8.8113e-14,\n -1.1234e-13, 1.6008e-13, 1.7094e-13, -7.3347e-13, 3.4083e-13,\n 5.3799e-13, 6.6348e-13, -1.2810e-12, 1.0864e-12, 4.9360e-13,\n 2.3979e-13, -6.9206e-14, -2.9314e-13, -1.4127e-13, 2.3717e-13,\n 7.8941e-15, 4.4136e-13, -2.9617e-14, 4.0470e-13, 7.6512e-13,\n -4.7595e-13, 4.0090e-13, -3.9537e-13, 3.8701e-13, -5.6073e-13,\n -9.7303e-14, -1.0039e-13, 1.6455e-13, 3.0020e-13, 5.7734e-13,\n 3.8541e-13, 1.0465e-13, -8.6602e-13, -1.2119e-12, -6.5385e-13,\n -1.8893e-13, 1.9824e-13, -3.7249e-13, -1.5879e-13, -3.5043e-13,\n 7.5717e-14, 2.2512e-13, 3.1838e-13, 1.4040e-13, -1.3802e-12,\n 7.3193e-13, 2.6133e-13, -8.0366e-14, 1.5176e-13, 4.0702e-13,\n 7.4002e-14, -7.1480e-13, 7.1058e-13, 2.0177e-13, 9.7145e-14,\n -1.4209e-13, -6.7915e-13, -8.3777e-13, -1.8337e-13, 9.6993e-13,\n 9.5078e-14, -1.4627e-14, -9.7001e-14, -5.4743e-13, -1.8835e-12,\n 5.4255e-13, 2.9200e-13, 1.6550e-13, -4.3823e-13, -9.1301e-13,\n 3.4922e-13, -5.0825e-14, 9.9538e-13, 2.2635e-13, -8.4776e-14,\n 5.7390e-13, 2.1169e-13, 3.8275e-13, -3.9396e-13, 4.7688e-13,\n -7.8111e-14, -5.4502e-13, 1.1362e-13, 1.9248e-14, -2.7550e-13,\n 3.9049e-13, 1.1429e-13, -3.2889e-14, 3.6925e-13, -5.8962e-13,\n 2.0507e-13, -4.3508e-13, -3.1134e-13, -1.3126e-13, -6.0018e-13,\n -3.5777e-13, 2.4137e-13, -1.6305e-12, 7.2543e-13, -1.6219e-13,\n -1.8720e-13, -5.7689e-14, -1.5273e-12, -8.8423e-13, 1.1258e-13,\n -9.5780e-13, -1.8186e-14, 7.4004e-13, -3.7616e-14, 2.6925e-13,\n -6.2626e-13, 3.9206e-14, 1.4991e-13, -8.4445e-13, 2.0778e-13,\n 4.1072e-13, -8.1892e-14, 6.7681e-15, -2.0353e-13, 2.6853e-14,\n -1.0830e-13, 3.6503e-14, -1.9154e-14, -1.1170e-13, 2.0804e-13,\n -3.0875e-13, -1.2793e-13, -2.2513e-13, -5.3420e-14, -2.8790e-13,\n -1.2192e-13, 1.5179e-13, -1.3861e-12, 5.4481e-13, 2.8857e-14,\n 5.1264e-13, 4.1992e-13, -3.8764e-13, -1.9565e-15, 1.2117e-13,\n 1.5925e-13, 4.4246e-13, -2.3298e-13, 6.3316e-14, -6.7759e-13,\n 2.4327e-13, 7.7923e-14, -3.1463e-13, 1.2426e-13, 1.7710e-13,\n -9.7502e-14, 9.0202e-14, 7.3911e-14, -1.0370e-13, 1.1790e-13,\n 2.8620e-13, 1.3841e-14, 2.1112e-13, -9.1450e-13, -5.9403e-14,\n 1.4819e-13, -1.9314e-14, 5.4301e-14, 7.3045e-14, 4.7169e-14,\n 1.3294e-13, 3.3058e-13, -1.1797e-13, -6.0461e-14, 2.9469e-13,\n -5.8267e-14, 5.0096e-13, 7.0292e-14, -2.4498e-13, 1.6457e-13,\n 4.2753e-14, 8.8042e-13, 2.3718e-14, 2.4211e-14, -2.2209e-13,\n -1.2724e-14, -1.0724e-13, -1.2732e-13, 5.3860e-14, -1.7389e-13,\n 1.1242e-12, -7.8199e-13, 7.1858e-13, 3.1898e-14, -2.2022e-13,\n -1.3979e-12, -2.3437e-13, 1.3379e-14, 2.1677e-13, 1.8736e-13,\n 1.4350e-13, 5.1785e-13, 5.2524e-13, 2.0668e-13, 9.8085e-14,\n -1.4472e-14, -7.1800e-13, 2.7583e-13, 9.6674e-14, -1.4211e-15,\n 1.0014e-13, 6.3193e-13, 2.0581e-13, -2.2143e-13, 7.0213e-14,\n 3.4988e-14, -2.3568e-13, -6.0456e-13, 1.5337e-13, 3.3464e-13,\n 9.0303e-13, 1.0729e-12, 1.4906e-13, -1.0082e-13, 1.2486e-13,\n 2.7797e-14, 2.2164e-13, -2.3914e-14, -1.0012e-13, -3.0274e-13,\n 1.8147e-13, 3.9845e-13, 1.3649e-13, 7.6361e-14, 1.0619e-13,\n -2.4487e-13, 1.2490e-13, 4.1356e-13, -2.1816e-13, -1.2496e-12,\n -1.5547e-12, 1.3130e-13, -5.2723e-13, 2.0790e-13, -3.9802e-13,\n 2.3545e-13, 1.1653e-13, 9.8395e-13, -1.1600e-13, 5.7222e-13,\n 1.0505e-13, 3.1810e-13, 1.5060e-13, 5.4879e-13, 1.8989e-13,\n 1.6221e-13, 5.9620e-16, -5.6333e-14, 2.1865e-13, -6.9423e-13,\n 6.7278e-14, -6.9003e-13, -2.5502e-13, -8.4154e-13, 2.7078e-13,\n 1.6422e-13, 7.5887e-14, -3.7645e-13, 3.4228e-13, -2.5654e-14,\n 4.8019e-14, 2.0961e-13, 1.6256e-12, 1.8560e-14, -2.0483e-13,\n 3.8462e-13, -8.9363e-14, 2.8011e-13, -1.2559e-13, -7.9904e-14,\n -7.5228e-13, 1.0318e-13, -1.6793e-12, 4.2929e-13, -6.3445e-13,\n -5.0908e-13, -6.7319e-13, -1.6244e-14, 8.3437e-14, 4.7372e-14,\n 7.2398e-13, -1.5864e-13, -1.2002e-13, 3.1611e-13, 2.0229e-14,\n 3.5637e-13, 1.9841e-13, -5.8296e-14, -8.1018e-15, 7.4294e-14,\n -1.6606e-13, 2.2819e-13, -5.5413e-13, 4.2530e-13, -4.0069e-13,\n 7.0772e-14, -1.0987e-12, -2.9775e-13, 8.3838e-14, -5.7772e-13,\n 3.9159e-13, 2.8318e-13, 2.0482e-13, -4.9641e-13, 1.8631e-13,\n -6.7466e-14, 1.4827e-13, 1.3149e-14, -2.1283e-13, 4.7147e-13,\n 1.8949e-13, -3.7637e-13, -1.8170e-12, -1.0020e-12, 3.5852e-13,\n 2.6503e-13, 4.7728e-14, -4.4534e-14, 5.9908e-13, -1.1185e-13,\n 8.7948e-14, 7.5190e-13, 6.9157e-13, 4.1338e-14, 1.8098e-15,\n 7.1372e-14, -3.3364e-13, -8.8228e-14, -3.2766e-13, 2.8309e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.2071e-08, 4.4914e-08, 7.9288e-09, 1.1459e-08, 3.6900e-09, 2.1310e-09,\n 5.2357e-06, 2.3358e-08, 3.3040e-09, 5.7700e-07, 4.4944e-09, 2.6130e-08,\n 5.2184e-08, 1.2365e-07, 1.6465e-06, 7.2941e-08, 1.8863e-07, 4.8123e-06,\n 1.5803e-07, 1.0546e-07, 9.3447e-09, 1.0468e-08, 2.9942e-09, 4.2372e-07,\n 2.4350e-06, 1.8229e-06, 3.1840e-06, 8.0363e-08, 9.9045e-08, 3.5909e-06,\n 4.2543e-07, 1.3053e-08, 1.7732e-07, 2.1929e-08, 9.5209e-07, 3.8015e-09,\n 8.1681e-08, 4.5558e-09, 5.8882e-08, 6.0267e-07, 6.4035e-08, 7.8876e-07,\n 5.7311e-09, 5.5161e-07, 3.6976e-07, 2.3568e-08, 5.2565e-07, 5.4697e-09,\n 1.4106e-08, 1.0963e-07, 1.0704e-05, 5.6598e-09, 8.4221e-08, 2.4884e-08,\n 4.0748e-09, 2.4312e-08, 4.6763e-07, 1.6132e-07, 1.5804e-07, 1.9591e-07,\n 3.5680e-06, 3.3550e-09, 3.8616e-08, 4.4713e-08, 3.3247e-06, 2.6496e-07,\n 2.8464e-08, 6.8214e-06, 2.0191e-06, 5.2009e-07, 1.4180e-08, 1.1741e-06,\n 6.3728e-07, 1.3079e-07, 1.8333e-09, 4.3000e-09, 8.3519e-07, 1.7943e-09,\n 4.1736e-07, 1.0896e-06, 3.2634e-08, 2.1727e-07, 9.6973e-07, 7.1152e-07,\n 3.7468e-06, 6.9341e-09, 6.4515e-07, 3.3868e-07, 2.2859e-08, 9.7405e-08,\n 2.0962e-09, 4.7059e-09, 1.0585e-07, 1.1364e-08, 6.5703e-09, 5.6641e-07,\n 8.8436e-08, 1.5711e-06, 5.0768e-09, 4.9326e-08, 1.3652e-09, 3.2417e-06,\n 1.5366e-07, 9.8529e-08, 2.1471e-06, 2.2390e-07, 4.5723e-09, 2.4736e-09,\n 5.7377e-08, 4.5422e-07, 1.1946e-08, 4.8148e-09, 7.6905e-07, 8.1076e-08,\n 4.9929e-09, 4.3690e-09, 8.7017e-08, 2.6626e-07, 1.5983e-07, 1.5688e-06,\n 8.8474e-08, 4.7856e-07, 6.3799e-09, 1.3720e-07, 7.5534e-08, 3.5928e-08,\n 1.6973e-06, 1.7854e-08, 3.1653e-09, 1.2001e-06, 6.3849e-09, 2.6422e-07,\n 4.0581e-06, 8.8225e-08, 3.5557e-06, 8.5209e-06, 2.8027e-08, 1.5900e-06,\n 6.2276e-07, 2.8379e-07, 8.0489e-08, 8.5822e-08, 1.2113e-08, 2.4446e-07,\n 7.7458e-07, 1.8444e-08, 1.9284e-06, 4.5685e-08, 2.3756e-08, 6.0349e-07,\n 1.1742e-07, 1.0461e-07, 2.5624e-08, 4.5185e-09, 8.3662e-09, 2.3801e-08,\n 3.6587e-09, 1.8371e-07, 8.8396e-08, 1.3845e-07, 3.9851e-08, 4.8908e-08,\n 1.5566e-08, 1.3541e-06, 4.5202e-07, 3.0241e-08, 1.9676e-09, 1.7074e-08,\n 5.5779e-09, 1.1543e-08, 4.6070e-07, 1.0517e-05, 3.4210e-07, 2.8961e-09,\n 5.5760e-07, 4.7684e-08, 1.4969e-06, 1.6915e-08, 3.7452e-08, 1.8644e-06,\n 3.2635e-08, 1.2316e-08, 1.2922e-06, 6.2640e-09, 5.8052e-07, 2.3287e-08,\n 1.8452e-06, 3.8267e-07, 7.3314e-07, 9.1271e-09, 3.0205e-09, 2.8984e-07,\n 1.7794e-07, 1.2251e-07, 2.5495e-06, 1.4004e-06, 5.3683e-09, 1.4613e-06,\n 2.4856e-08, 5.5826e-07, 4.0462e-08, 5.6713e-07, 1.4174e-06, 4.5516e-07,\n 2.1022e-06, 4.8379e-08, 5.1328e-08, 1.9786e-07, 1.8849e-07, 8.6064e-07,\n 1.9897e-07, 8.7753e-08, 2.2030e-07, 8.2328e-07, 3.0978e-06, 1.5317e-07,\n 3.6397e-09, 6.6739e-07, 1.0914e-06, 7.8175e-08, 6.7220e-09, 1.5011e-07,\n 4.1243e-06, 3.2810e-08, 9.1160e-07, 7.8839e-08, 1.2036e-06, 6.7757e-08,\n 3.8721e-09, 2.2013e-09, 9.8341e-08, 1.2422e-09, 1.4124e-06, 6.4555e-07,\n 4.4012e-07, 3.8500e-08, 7.1675e-08, 2.8142e-08, 2.1192e-07, 2.2007e-06,\n 1.3678e-08, 1.7385e-06, 6.0993e-08, 2.2253e-07, 3.3411e-08, 8.0224e-07,\n 1.5063e-06, 1.6982e-08, 9.0964e-09, 1.2236e-08, 1.9780e-06, 9.9206e-07,\n 1.9705e-08, 3.4033e-06, 1.4960e-08, 1.6832e-08, 8.4176e-09, 2.2486e-09,\n 1.0218e-06, 3.4923e-07, 3.6625e-08, 4.0961e-07, 5.0747e-07, 5.5334e-08,\n 2.5321e-06, 2.7767e-08, 9.0582e-09, 1.6962e-07, 4.3988e-09, 4.2358e-08,\n 5.1660e-09, 1.0403e-07, 2.8134e-07, 5.3478e-08, 1.7019e-06, 4.3609e-08,\n 6.9416e-08, 3.1923e-07, 4.7004e-06, 1.5667e-08, 1.0055e-06, 8.7598e-07,\n 2.5407e-08, 4.3528e-07, 3.9696e-09, 7.5948e-08, 6.6432e-08, 1.9826e-07,\n 1.0772e-07, 1.6727e-06, 1.0034e-06, 8.5815e-07, 6.9499e-09, 1.6877e-07,\n 1.9955e-07, 5.0575e-07, 6.0725e-06, 1.1960e-08, 4.7587e-07, 1.9421e-07,\n 5.7996e-07, 4.1784e-09, 1.7742e-06, 9.0095e-06, 2.8207e-09, 6.8239e-08,\n 5.9241e-07, 2.0943e-08, 1.0384e-08, 7.2014e-08, 1.1856e-07, 3.2696e-06,\n 6.6188e-07, 1.0041e-07, 6.4723e-06, 8.7233e-09, 9.1574e-08, 1.0501e-08,\n 1.7339e-07, 6.6302e-08, 6.1632e-09, 4.6422e-07, 2.4032e-07, 4.0579e-07,\n 6.0100e-08, 3.6470e-09, 1.8074e-06, 1.3453e-07, 5.4686e-07, 2.7012e-06,\n 7.5262e-09, 2.1259e-09, 1.1046e-06, 1.5993e-06, 1.2585e-06, 8.6208e-09,\n 4.0266e-07, 5.0577e-07, 1.7975e-06, 2.0621e-06, 1.3695e-06, 1.6591e-07,\n 1.4947e-07, 2.9037e-06, 3.4641e-06, 5.1410e-09, 6.1184e-08, 6.3799e-09,\n 1.0678e-08, 5.8519e-07, 5.5217e-07, 1.4851e-08, 1.4064e-09, 4.1667e-07,\n 2.8591e-09, 3.9355e-06, 1.0664e-06, 4.5416e-08, 4.4762e-07, 4.7635e-08,\n 2.3345e-08, 5.1173e-06, 6.7163e-07, 2.9296e-07, 2.4004e-06, 1.4822e-07,\n 7.9522e-08, 1.5077e-08, 2.0384e-06, 9.3939e-09, 4.0914e-06, 2.2069e-06,\n 3.2221e-09, 2.8273e-06, 1.6593e-06, 9.4018e-09, 3.4458e-06, 1.9588e-06,\n 3.0834e-06, 4.5087e-08, 2.4950e-06, 8.7112e-09, 1.0180e-06, 1.9206e-08,\n 2.3551e-08, 1.6822e-08, 2.1084e-08, 9.6102e-09, 1.2551e-06, 1.2726e-07,\n 5.2332e-07, 1.9461e-07, 6.3361e-08, 8.4166e-09, 6.6510e-09, 4.8330e-08,\n 1.4508e-07, 8.6535e-07, 1.2761e-06, 1.7228e-08, 5.2836e-08, 4.3131e-07,\n 3.4119e-06, 5.5730e-06, 5.3722e-07, 2.4283e-07, 3.3002e-08, 1.1497e-07,\n 2.1309e-06, 6.9738e-07, 4.6122e-09, 4.0221e-08, 6.0141e-08, 2.4560e-08,\n 5.3625e-06, 3.0134e-06, 8.0441e-07, 3.7228e-08, 1.1008e-07, 1.2465e-08,\n 2.8224e-08, 2.2044e-08, 4.8127e-09, 7.1724e-07, 1.2814e-08, 3.4578e-08,\n 2.3645e-08, 2.5192e-08, 1.1312e-08, 8.1573e-08, 8.6529e-08, 2.8456e-06,\n 4.0660e-07, 7.9809e-09, 1.3161e-07, 6.6659e-08, 4.7032e-07, 3.3603e-07,\n 4.1558e-07, 4.5356e-08, 1.2035e-07, 5.2909e-07, 1.0521e-06, 8.4303e-09,\n 9.0134e-07, 2.3699e-09, 5.4811e-06, 1.7718e-07, 1.2388e-08, 2.3515e-07,\n 5.3983e-09, 1.6063e-09, 1.9065e-08, 3.0175e-09, 4.3993e-08, 7.1829e-07,\n 3.1636e-09, 4.5270e-08, 8.5549e-07, 1.2867e-07, 2.2045e-09, 1.4803e-08,\n 8.7140e-08, 5.6303e-07, 3.3886e-07, 3.2430e-09, 9.8407e-07, 4.0708e-07,\n 3.7207e-09, 8.3238e-09, 1.4945e-08, 2.6987e-08, 2.3006e-07, 2.3020e-09,\n 7.7535e-09, 1.8577e-08, 2.9469e-07, 4.6178e-08, 6.1063e-08, 3.8310e-06,\n 7.9718e-09, 6.7952e-08, 2.3115e-08, 3.2381e-07, 1.6596e-07, 4.4641e-08,\n 1.4461e-08, 1.1815e-08, 1.1089e-08, 1.0842e-08, 2.3137e-06, 4.2620e-06,\n 6.0455e-07, 1.9293e-07, 3.0416e-06, 8.4424e-07, 2.4116e-09, 2.9520e-08,\n 6.0428e-08, 1.3826e-07, 1.0033e-06, 6.2908e-06, 1.2899e-08, 2.9322e-06,\n 9.6375e-09, 2.5494e-07, 9.4960e-07, 3.1264e-06, 2.9747e-07, 5.5673e-09,\n 6.2957e-08, 1.7667e-07, 1.1985e-08, 3.3460e-07, 1.0398e-07, 8.4989e-08,\n 1.0833e-07, 4.6476e-08, 4.6518e-06, 5.1669e-08, 4.0889e-06, 5.1832e-07,\n 5.3165e-09, 2.3076e-07, 2.8459e-08, 1.8223e-08, 4.3464e-07, 4.1415e-07,\n 4.7338e-08, 4.8682e-08, 1.0788e-08, 4.4871e-06, 1.8217e-08, 2.2183e-09,\n 2.5966e-06, 1.6261e-08, 3.6099e-09, 2.0567e-07, 1.5235e-08, 1.0246e-08,\n 8.6848e-07, 6.1489e-09, 1.2166e-07, 8.3661e-08, 5.6511e-07, 2.4844e-09,\n 2.2900e-09, 8.0245e-06, 5.1122e-09, 2.9891e-09, 7.0954e-08, 3.1505e-09,\n 1.4691e-06, 7.6686e-09, 1.0638e-06, 1.8300e-08, 6.4669e-08, 1.5287e-08,\n 5.1393e-07, 5.3024e-07, 3.9690e-09, 3.5042e-06, 1.9942e-09, 3.5777e-06,\n 4.2835e-06, 2.1875e-07, 3.5788e-08, 1.2305e-07, 1.6510e-06, 3.7393e-07,\n 3.7869e-07, 5.1808e-07, 7.1671e-07, 2.1232e-09, 1.0686e-07, 4.5494e-07,\n 6.5882e-08, 1.1639e-08, 7.7365e-08, 3.1075e-06, 3.7694e-08, 1.5483e-06,\n 3.0031e-07, 2.1406e-06, 2.0002e-08, 6.6408e-08, 7.5092e-08, 2.5837e-09,\n 8.3217e-09, 5.4322e-09, 1.4389e-06, 1.7385e-08, 1.6924e-07, 5.0386e-07,\n 6.4018e-08, 5.2418e-07, 1.0489e-07, 3.9581e-08, 1.3101e-08, 1.5758e-07,\n 9.7913e-07, 5.1402e-09, 1.7767e-08, 3.5892e-08, 3.1287e-06, 1.1603e-08,\n 2.8017e-08, 2.9523e-08, 5.0832e-08, 4.1319e-07, 4.2237e-08, 2.6315e-07,\n 2.1255e-07, 3.1563e-07, 1.0054e-06, 5.9389e-08, 1.8577e-07, 1.5460e-07,\n 1.7462e-07, 8.8161e-08, 7.9183e-08, 8.1351e-07, 5.5775e-08, 1.1260e-07,\n 1.4320e-06, 7.3088e-08, 4.7963e-07, 4.1296e-08, 8.0299e-08, 4.2229e-06,\n 1.7336e-08, 1.4915e-06, 2.2503e-06, 2.3550e-07, 8.5850e-09, 2.6881e-08,\n 1.9543e-07, 9.1169e-08, 6.4435e-08, 9.3572e-08], device='cuda:0')" + }, + "32": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.7270e-16, -2.3300e-13, 1.5534e-16, 9.5774e-16, 1.9246e-15,\n -7.5518e-16, -1.8110e-14, -3.9829e-15, -2.3616e-15, -5.1314e-15,\n 1.7448e-16, -6.3087e-17, 8.9856e-16, -1.8770e-15, 4.6858e-16,\n -1.0255e-13, -3.6939e-14, -2.2554e-15, 1.1171e-15, -4.1107e-16,\n -1.0282e-15, -3.2668e-14, -2.0963e-15, 1.5010e-14, -1.5265e-14,\n -7.3815e-15, -1.2168e-14, -2.8425e-16, -7.7020e-15, -3.5917e-14,\n 5.7193e-17, 1.2138e-15, 1.0886e-15, 3.8572e-15, -4.2684e-16,\n -7.2113e-16, 1.8227e-15, 1.7974e-16, -5.7040e-15, 9.4258e-16,\n -5.9911e-16, -2.1397e-15, -4.0315e-15, -1.6656e-14, -2.6918e-14,\n -5.3115e-16, -4.1060e-14, -8.8981e-15, -4.5282e-16, -1.8430e-14,\n -5.1581e-15, 2.7960e-16, 2.3677e-15, 2.3022e-15, -2.3175e-15,\n -2.4594e-15, -1.2338e-16, -1.4845e-15, 1.3213e-16, -4.2962e-15,\n -2.5999e-14, -3.3297e-15, 3.0549e-18, -8.5682e-15, -6.4594e-15,\n -1.3077e-14, -2.1355e-15, 8.0577e-15, -4.3889e-15, -5.4383e-16,\n -5.8495e-15, 4.8011e-15, -6.3686e-15, 2.2610e-14, -8.8123e-16,\n -2.6813e-15, -4.5058e-14, -1.2441e-15, -1.7402e-14, 4.1600e-15,\n -3.5559e-15, -4.0245e-14, 2.5877e-15, -2.1769e-15, 1.4546e-15,\n 1.5985e-15, -9.1152e-15, -3.1922e-14, -7.3273e-16, 8.6659e-16,\n 1.3115e-16, -3.7908e-15, -4.1322e-14, -1.6265e-15, 2.6934e-15,\n -1.9231e-14, -6.7729e-14, 1.7456e-16, -1.3609e-16, -1.8740e-16,\n -3.2334e-15, 5.1665e-15, -5.6735e-14, -5.3944e-14, -1.7170e-14,\n -2.1065e-14, -2.6527e-15, -2.7706e-16, 2.2417e-16, -1.8112e-14,\n -3.8691e-15, -2.5696e-15, 6.0224e-16, 5.3597e-16, -1.0998e-15,\n 6.8373e-17, 5.8545e-16, 7.7026e-15, -6.2736e-15, -5.6523e-15,\n 1.1983e-15, -1.7079e-15, -6.5029e-15, 1.1552e-15, -1.9733e-14,\n -2.7886e-15, -3.0730e-14, -6.2101e-16, 2.3433e-15, -2.7713e-14,\n -3.5523e-16, 5.1517e-16, 1.1716e-15, 7.7733e-16, -3.2081e-15,\n -5.4939e-15, 1.5021e-15, -1.1991e-14, 3.9181e-16, -6.0478e-16,\n -1.2013e-16, -6.5612e-14, 1.0922e-14, 1.7035e-14, -8.0497e-15,\n -4.8913e-16, -1.5557e-14, 1.1814e-15, 3.2407e-16, 8.9257e-16,\n -1.2438e-15, -8.8984e-16, 1.0218e-15, -1.6897e-14, -7.6666e-16,\n -3.2464e-16, 2.9858e-15, -2.1527e-15, -4.0041e-15, 1.3071e-15,\n -4.2091e-16, 2.4288e-15, -4.7504e-15, -3.3262e-16, -4.2652e-16,\n 1.8940e-15, -1.9861e-15, -3.5718e-15, -9.7354e-16, -4.5162e-16,\n 3.0822e-15, 5.8971e-16, 1.9007e-16, 8.1815e-16, -9.5680e-16,\n -7.3033e-16, -2.2416e-14, -9.1323e-16, 4.2791e-15, -2.0961e-14,\n -5.2588e-14, 6.8576e-16, -3.9536e-15, 2.9168e-16, -6.9189e-14,\n 7.2718e-16, -2.9995e-14, 5.0440e-16, -3.3621e-14, -4.2573e-15,\n 1.7209e-16, -7.4299e-15, -5.2802e-16, 3.6243e-17, -6.2896e-16,\n -3.4236e-15, 1.1819e-15, -1.6852e-14, 1.4663e-15, 6.6396e-15,\n -5.4956e-15, -5.9241e-15, 2.7189e-15, -1.2105e-14, -1.0007e-15,\n -3.2004e-16, -4.5259e-15, -1.6048e-14, 9.1616e-16, -1.3545e-15,\n -2.9722e-16, -2.3724e-16, -5.8968e-16, -5.4611e-15, 2.6009e-15,\n -6.0059e-15, 4.8127e-15, -9.2835e-16, 7.0218e-16, 2.4571e-15,\n 7.6943e-16, 8.5649e-15, -6.2535e-15, 4.6128e-16, -3.8114e-15,\n 1.4971e-15, -6.8898e-15, -3.9732e-14, -2.1750e-14, 6.4761e-16,\n 3.8992e-15, 9.5242e-16, 2.8627e-15, -2.7095e-14, 2.4225e-15,\n -2.1852e-16, -2.3318e-17, 1.4304e-15, 6.2460e-16, 1.1428e-14,\n 4.7266e-16, 1.4196e-15, -1.8275e-16, 1.3636e-15, -3.2442e-16,\n -2.9883e-16, -5.8046e-15, 1.4535e-15, 1.6472e-15, -1.4555e-15,\n -2.0385e-14, 5.0809e-15, -2.6318e-16, -4.4638e-15, -1.9127e-15,\n 1.1520e-16, -2.0774e-15, 1.0104e-15, -1.0463e-15, -7.2075e-16,\n -4.2206e-16, 1.1696e-15, -2.6627e-16, -1.0824e-15, 2.9704e-15,\n -6.3481e-15, 1.2882e-15, -3.9332e-14, -1.4406e-16, -6.9812e-15,\n -4.8850e-16, -5.3677e-16, -3.2356e-15, -1.5607e-14, -3.3964e-15,\n -9.9890e-16, -1.2702e-13, -4.3845e-15, 3.8668e-15, -8.4134e-16,\n 2.2703e-15, -8.2808e-15, -7.1270e-16, 2.1894e-15, 3.1343e-15,\n -2.5725e-13, 1.0823e-15, -1.2032e-14, 1.7382e-15, -5.3183e-15,\n -2.3343e-14, -3.9690e-15, -6.4757e-15, 1.6277e-14, 2.9318e-16,\n -1.0669e-15, -1.7044e-14, -5.1251e-15, 1.8392e-15, -2.8586e-15,\n -1.6942e-14, 1.2297e-16, -4.8266e-15, -1.2596e-15, -7.3925e-16,\n -2.8512e-15, 4.6740e-15, -2.1662e-16, -4.0995e-14, -2.5783e-14,\n -1.5666e-14, 1.0350e-15, -3.0563e-14, 1.4890e-14, 3.2777e-16,\n -1.3584e-14, -1.5897e-15, -4.1837e-14, -2.1110e-15, -1.0572e-15,\n -1.0047e-16, -3.0562e-16, 1.0008e-15, 1.2152e-15, -2.3322e-14,\n -4.6412e-15, -7.1723e-15, -5.5024e-15, -2.8973e-15, -8.0649e-15,\n -6.0563e-18, 6.1357e-16, -2.4762e-15, -7.3197e-15, 1.6934e-15,\n -5.7469e-15, -1.6878e-16, -1.8766e-14, -1.9950e-14, -6.5337e-14,\n -9.5179e-15, 1.7156e-15, -1.3800e-14, -1.1256e-14, -3.0519e-14,\n 5.7174e-16, 1.7142e-16, -1.2882e-15, 3.6379e-16, -6.0464e-14,\n 1.9063e-15, 8.2857e-16, 5.4168e-16, -2.5035e-15, -3.3220e-16,\n -1.2573e-14, -2.6882e-14, 2.6958e-15, -7.3596e-15, 1.1843e-15,\n -5.3825e-15, -2.7081e-14, -4.7256e-14, 1.1469e-15, 4.7762e-15,\n -2.7057e-16, -3.4803e-15, 9.2991e-16, -1.0263e-14, -5.0541e-14,\n -1.7453e-16, -1.8983e-15, 5.9590e-16, -2.2954e-14, -1.7431e-14,\n 1.2945e-15, -1.2200e-14, 3.0047e-15, -1.1736e-14, 2.5628e-15,\n -1.1300e-14, -4.6766e-16, -2.4720e-15, -5.1570e-15, -1.5290e-14,\n -1.0135e-15, -2.5783e-15, 6.1687e-16, -3.8655e-15, -2.9543e-14,\n -1.7740e-17, 2.8873e-16, -3.8699e-15, 8.9328e-16, -9.6188e-15,\n -1.2787e-14, -6.2363e-15, 2.4863e-15, 1.9825e-15, -3.3325e-14,\n -1.8336e-14, -1.4935e-14, -5.2837e-14, -6.2363e-16, -2.1173e-14,\n -7.8980e-15, -1.0481e-15, -9.0968e-14, -1.6868e-14, -1.5303e-17,\n -7.9720e-14, -5.2384e-16, -5.2745e-14, 6.5433e-16, -1.3000e-14,\n -1.8943e-14, -4.1222e-15, -7.8833e-16, -9.0258e-14, 6.5976e-16,\n -1.0308e-15, -8.6489e-18, -2.8167e-16, -1.8110e-14, 5.9911e-16,\n -1.2084e-15, -3.9319e-14, -1.0567e-15, -1.5712e-15, -4.1929e-16,\n -1.7581e-14, -7.3100e-15, -9.6746e-16, 1.3771e-15, -7.9952e-15,\n 1.0400e-15, -1.1200e-15, -2.5674e-14, -4.4982e-16, -3.6475e-15,\n -3.2345e-15, 2.5405e-16, -1.2587e-14, -2.0528e-16, -1.4270e-15,\n 1.7071e-16, -6.3308e-15, -4.8684e-15, -4.5484e-16, -4.4758e-15,\n -8.0556e-16, 3.4380e-16, -1.7113e-14, 1.8825e-15, 9.6125e-16,\n 1.1871e-15, -5.9314e-16, 6.8505e-16, -1.0459e-14, 4.1767e-16,\n -8.8938e-16, 3.2045e-17, -2.1343e-14, -2.3150e-14, 1.5336e-15,\n -1.9831e-17, 3.2259e-15, -1.0693e-16, 4.5211e-16, -1.0596e-14,\n -1.2425e-16, -7.2731e-16, 2.8335e-15, -3.6418e-15, 7.9696e-16,\n 7.1017e-16, -2.6063e-15, 4.9754e-16, 8.1908e-16, -6.7187e-16,\n -1.6015e-15, -9.9874e-15, 8.2522e-16, -5.3861e-16, -1.2400e-14,\n 1.5911e-15, -1.5781e-15, -1.7329e-15, 7.8805e-17, -9.2401e-15,\n 8.6493e-15, -1.0305e-14, -8.4360e-15, 1.8965e-16, -4.0277e-15,\n -6.1784e-14, -2.0485e-15, -4.3454e-18, -5.7747e-16, -6.5360e-15,\n -3.7692e-16, -2.8652e-15, -4.4098e-17, 4.3402e-16, 7.9457e-16,\n 5.6279e-16, -3.4862e-14, -1.5917e-15, 2.6311e-16, 5.9761e-16,\n -1.5190e-16, 9.9704e-16, 6.5255e-16, -2.2956e-15, 3.6795e-16,\n -8.8878e-16, -1.8353e-14, 8.4242e-15, 1.6774e-16, 3.0033e-16,\n 9.8924e-16, 9.7929e-15, -7.0877e-17, -1.8982e-15, 5.1209e-16,\n -3.1888e-15, -7.7042e-15, -3.1484e-15, 3.0521e-16, -1.6951e-15,\n -1.2156e-14, 7.9780e-16, 1.2707e-15, 7.8500e-16, -5.5338e-15,\n -4.0805e-15, 3.0024e-16, -4.7579e-16, -2.1962e-14, -1.9839e-14,\n -2.5687e-14, -9.4510e-16, -1.9039e-14, -2.2265e-15, -4.8356e-14,\n -1.0751e-14, 7.2116e-17, 5.7475e-15, 9.3109e-17, -4.7645e-16,\n -3.0627e-15, 1.2208e-16, 6.2771e-17, 8.4283e-16, 1.8537e-16,\n 4.1603e-17, -1.8895e-14, -1.3328e-15, -2.0558e-15, -1.6913e-14,\n 9.3017e-16, -1.0491e-14, -1.0236e-15, -2.3277e-14, 1.4757e-16,\n -1.1703e-14, 2.4663e-16, -5.3219e-15, -2.1391e-14, 1.2342e-15,\n -4.0569e-15, -2.1957e-15, 1.4599e-14, -1.5032e-14, -5.3807e-14,\n -6.3124e-15, -6.9376e-16, 3.8454e-16, 2.4440e-15, 2.1991e-15,\n -5.5914e-15, 2.0308e-16, -3.8670e-14, -3.8657e-15, -1.5324e-14,\n -1.3518e-14, -7.3390e-14, -6.8803e-16, 4.9804e-18, -1.3465e-15,\n 1.6485e-15, -7.6671e-16, -5.1371e-15, 2.7457e-16, -5.2716e-15,\n 3.9479e-15, -5.7304e-16, 2.3938e-15, 1.3825e-15, -7.5002e-16,\n -2.1198e-14, 5.0352e-16, -9.0167e-14, 3.7735e-16, -1.4358e-14,\n -1.3935e-15, -8.8848e-14, -4.1285e-15, -1.9202e-15, -4.5939e-15,\n 9.8759e-16, -1.6253e-16, 6.9664e-16, -4.2131e-15, 2.7856e-16,\n -2.1660e-15, -9.1028e-14, 4.1607e-16, -1.6951e-14, -9.4739e-16,\n -2.4291e-16, -8.5544e-15, -7.8939e-14, -1.8598e-14, -5.3639e-15,\n -2.4151e-15, -1.3914e-14, -1.7156e-15, 1.5685e-15, 3.4162e-15,\n 3.9392e-16, 1.9202e-15, 1.7166e-15, -8.7909e-15, -1.8867e-15,\n -1.9585e-14, -8.8465e-15, 1.9143e-15, -4.1894e-16, 1.5945e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.3279e-11, 9.6410e-12, 1.6484e-13, 1.4483e-13, 5.3480e-13, 1.7637e-14,\n 2.4699e-09, 1.7660e-11, 1.2838e-12, 2.7230e-11, 8.2974e-13, 9.3611e-13,\n 4.4889e-13, 1.7189e-10, 3.9549e-10, 3.5978e-11, 1.4555e-12, 1.2574e-09,\n 9.5209e-13, 9.8933e-12, 1.8724e-12, 2.4567e-13, 6.1274e-13, 2.5857e-11,\n 1.9283e-10, 2.6048e-10, 4.7672e-10, 4.4128e-12, 6.1395e-13, 1.8205e-09,\n 3.8064e-12, 4.0382e-12, 3.3273e-12, 3.3270e-12, 6.7148e-11, 9.1370e-13,\n 3.4504e-12, 1.0804e-13, 2.7936e-11, 3.6640e-11, 3.0977e-13, 2.8836e-11,\n 4.7561e-13, 5.5703e-11, 1.3812e-10, 7.9970e-12, 1.9885e-11, 6.6984e-14,\n 3.1497e-12, 5.2913e-12, 3.4726e-09, 2.0142e-13, 4.3033e-13, 5.8478e-13,\n 2.1404e-13, 5.9412e-12, 4.3379e-12, 2.0680e-12, 8.5689e-13, 5.7435e-12,\n 1.5168e-09, 7.0113e-14, 1.0091e-11, 2.3204e-13, 5.9406e-10, 1.9568e-10,\n 8.0992e-12, 2.8730e-09, 8.5703e-11, 4.0947e-12, 3.2270e-12, 2.8486e-10,\n 4.6695e-11, 6.5450e-11, 9.4180e-14, 2.1460e-13, 2.0504e-11, 1.0693e-13,\n 1.2240e-10, 1.3259e-10, 1.7336e-13, 3.8046e-11, 2.4590e-10, 9.1854e-12,\n 8.8364e-10, 1.1952e-13, 3.8144e-11, 1.2647e-10, 2.2283e-13, 5.4930e-12,\n 4.5253e-14, 1.4244e-13, 7.8198e-11, 6.4987e-12, 1.1213e-13, 3.2803e-11,\n 2.8086e-11, 4.8691e-11, 8.9315e-13, 3.1344e-11, 1.2597e-13, 2.7318e-09,\n 4.0284e-11, 6.6479e-11, 4.7682e-10, 1.2460e-10, 2.6219e-13, 7.5378e-14,\n 2.7546e-12, 8.3410e-11, 8.4906e-12, 3.1917e-12, 9.9804e-12, 7.4566e-13,\n 1.9637e-13, 3.8744e-14, 1.4879e-11, 6.1283e-12, 1.2487e-10, 5.1196e-10,\n 2.2458e-11, 1.2766e-11, 4.9182e-13, 8.8711e-13, 6.3110e-13, 1.4160e-11,\n 2.3129e-10, 5.9476e-13, 1.7247e-13, 5.4036e-10, 1.9191e-13, 2.0174e-12,\n 2.8219e-10, 1.3981e-12, 4.1461e-10, 6.7016e-10, 5.3422e-13, 5.7411e-10,\n 9.8130e-12, 6.3228e-12, 7.8577e-12, 2.2630e-11, 1.5938e-12, 2.4473e-10,\n 2.3800e-10, 3.4332e-13, 9.4445e-10, 1.2023e-11, 1.4727e-13, 4.2011e-12,\n 6.3618e-12, 7.9367e-13, 9.8467e-12, 1.2284e-13, 8.7784e-14, 5.2848e-13,\n 5.2635e-13, 4.3490e-12, 1.9438e-11, 7.5906e-11, 1.1706e-11, 1.5160e-12,\n 9.3590e-12, 1.8817e-10, 3.6370e-12, 1.7298e-12, 6.3320e-14, 7.6349e-12,\n 3.3524e-12, 6.4876e-13, 2.9150e-12, 1.3635e-09, 1.2035e-11, 2.8638e-14,\n 2.9523e-12, 7.1307e-13, 2.4710e-10, 3.3261e-13, 7.8154e-13, 3.2472e-10,\n 1.0961e-12, 2.9582e-13, 1.3890e-10, 6.2323e-14, 3.9554e-11, 4.6641e-13,\n 3.7850e-10, 3.8564e-11, 1.9622e-10, 2.7202e-13, 2.4473e-13, 2.1452e-11,\n 1.1637e-12, 2.2183e-12, 2.3937e-10, 2.0186e-10, 1.1046e-12, 5.7162e-10,\n 6.9046e-12, 2.7440e-10, 5.3506e-13, 6.0631e-11, 1.0952e-10, 3.1505e-12,\n 8.0618e-11, 5.2319e-13, 2.7825e-11, 9.4652e-12, 1.1168e-12, 3.5216e-12,\n 3.1813e-12, 2.8049e-12, 1.1229e-12, 2.7092e-10, 6.4666e-10, 1.1017e-12,\n 6.8207e-14, 9.4842e-11, 1.4864e-10, 2.3244e-11, 1.7394e-13, 2.8445e-11,\n 2.7275e-09, 1.9838e-11, 2.3350e-10, 8.0239e-13, 9.5849e-12, 9.0258e-13,\n 7.0621e-14, 4.4232e-13, 2.5861e-11, 4.6303e-13, 2.3477e-10, 3.1720e-10,\n 4.5162e-12, 8.3813e-12, 1.7125e-12, 9.3815e-12, 3.5709e-11, 1.5960e-09,\n 1.2705e-13, 8.8324e-11, 2.1837e-12, 1.8110e-12, 3.0263e-13, 2.8954e-10,\n 2.9601e-10, 3.1940e-12, 1.7605e-13, 3.0022e-12, 5.6471e-10, 9.5684e-10,\n 2.2353e-13, 1.6360e-09, 3.2995e-13, 2.1549e-12, 5.3615e-12, 6.2060e-14,\n 6.8332e-11, 4.2807e-12, 5.8888e-13, 1.4204e-10, 5.0827e-12, 7.6570e-12,\n 1.0762e-09, 1.8463e-11, 1.7888e-12, 4.0990e-11, 7.4741e-13, 2.3750e-11,\n 6.9148e-13, 1.9308e-11, 9.3303e-12, 7.2050e-13, 1.8192e-10, 1.6980e-13,\n 1.9653e-11, 5.5997e-11, 2.6928e-09, 3.6569e-13, 7.9391e-12, 1.3595e-11,\n 1.4509e-13, 1.4643e-12, 7.6149e-14, 1.9221e-11, 2.0125e-12, 1.7718e-12,\n 1.5775e-12, 5.6909e-10, 1.3952e-10, 1.2703e-11, 1.3429e-12, 1.0058e-10,\n 1.7209e-12, 1.6257e-11, 1.5324e-09, 6.2887e-12, 1.0964e-10, 1.4634e-11,\n 1.5355e-11, 1.4015e-12, 6.8576e-11, 2.2558e-09, 1.4803e-13, 1.4734e-11,\n 2.7129e-10, 3.6239e-13, 7.5443e-13, 3.9144e-12, 1.1319e-11, 2.4234e-10,\n 1.3202e-11, 2.5263e-12, 1.5507e-09, 9.8464e-14, 5.1687e-13, 4.3991e-13,\n 1.2988e-11, 1.0271e-12, 2.1367e-13, 1.4259e-11, 4.4335e-12, 4.7336e-12,\n 5.9009e-12, 1.8483e-12, 7.5999e-10, 6.6879e-11, 2.8697e-11, 5.7741e-11,\n 6.8202e-14, 5.7624e-13, 3.2058e-10, 9.3270e-10, 7.8454e-11, 8.5187e-14,\n 5.2002e-12, 5.6487e-11, 2.1220e-10, 1.0374e-09, 3.0131e-10, 6.3303e-12,\n 1.0334e-11, 2.1846e-09, 1.4086e-09, 3.7433e-14, 4.3844e-13, 9.6403e-13,\n 1.3948e-12, 4.8479e-12, 4.3737e-10, 1.8339e-12, 1.1135e-13, 9.1338e-11,\n 1.1233e-13, 6.2647e-10, 2.8343e-10, 6.8046e-13, 5.0763e-11, 1.8994e-13,\n 8.7155e-13, 2.9406e-09, 1.0017e-09, 1.7808e-11, 4.8821e-10, 1.7643e-12,\n 9.2546e-12, 1.1925e-13, 1.2293e-10, 1.9608e-12, 6.2193e-10, 1.9629e-10,\n 3.8845e-13, 1.5401e-09, 2.0092e-10, 2.0052e-13, 1.3872e-09, 1.3459e-10,\n 1.2959e-09, 1.7287e-11, 6.6012e-10, 1.8080e-12, 1.6318e-10, 5.0987e-13,\n 5.5357e-13, 5.1372e-12, 4.4246e-13, 1.9760e-12, 6.3599e-11, 7.2617e-12,\n 1.1962e-11, 5.8755e-12, 9.9846e-13, 4.5484e-13, 1.7246e-13, 2.2167e-12,\n 9.6010e-13, 1.0585e-10, 8.7364e-12, 7.9377e-12, 4.4957e-11, 1.8509e-10,\n 5.4852e-10, 4.6042e-10, 1.1683e-10, 3.7185e-12, 9.2427e-13, 7.8621e-12,\n 3.2019e-11, 1.7500e-11, 4.5173e-13, 1.0201e-12, 2.1975e-11, 1.2573e-12,\n 3.0751e-10, 2.3071e-10, 4.2193e-11, 1.3791e-11, 7.3693e-12, 2.4047e-12,\n 1.5833e-12, 1.5560e-13, 1.4217e-13, 2.0659e-10, 1.6910e-13, 4.1362e-13,\n 8.9281e-13, 3.6095e-13, 6.5640e-12, 6.3926e-13, 5.0238e-12, 2.9213e-10,\n 1.2305e-10, 9.3674e-14, 1.6847e-12, 7.9526e-13, 2.2031e-10, 2.0934e-12,\n 1.4442e-11, 2.1508e-11, 1.9302e-11, 2.7808e-11, 2.2700e-10, 9.9705e-14,\n 3.3332e-11, 1.2193e-13, 1.3397e-09, 1.6347e-12, 4.7576e-12, 3.9375e-12,\n 3.9588e-13, 1.5389e-13, 8.8033e-13, 4.5156e-13, 4.8152e-12, 1.3771e-10,\n 2.2773e-13, 1.4577e-13, 5.2883e-10, 4.3023e-12, 4.0408e-13, 7.8730e-13,\n 5.2075e-12, 8.9525e-11, 2.5667e-12, 1.5639e-13, 9.6564e-11, 1.4395e-11,\n 2.8811e-13, 6.1343e-13, 4.0937e-12, 3.1932e-13, 6.6973e-11, 4.7150e-13,\n 1.8486e-13, 1.0518e-11, 1.0501e-10, 5.3148e-13, 3.1259e-11, 5.1881e-10,\n 9.2952e-14, 4.1999e-12, 6.6108e-12, 6.1443e-12, 3.8661e-12, 2.4060e-11,\n 5.2468e-12, 6.6140e-12, 9.1422e-14, 6.6447e-12, 8.1359e-10, 3.5205e-10,\n 1.8881e-10, 7.8597e-11, 4.4101e-10, 1.9155e-10, 6.3030e-13, 9.5658e-12,\n 5.8618e-13, 7.7837e-12, 8.7517e-11, 1.6908e-09, 3.2449e-13, 3.4491e-10,\n 2.8390e-13, 1.0751e-11, 1.8798e-10, 4.2238e-10, 2.9105e-11, 2.5255e-13,\n 9.6312e-13, 6.4647e-12, 8.1937e-13, 1.6910e-12, 3.6886e-11, 1.6420e-11,\n 1.0215e-11, 3.5589e-12, 2.8933e-10, 9.4154e-13, 3.0251e-10, 9.2942e-11,\n 9.3267e-13, 1.2590e-11, 3.6080e-12, 2.5929e-13, 1.6916e-10, 2.0375e-10,\n 2.4464e-12, 7.5206e-13, 4.3376e-13, 1.1605e-10, 1.9104e-13, 8.2241e-13,\n 2.1392e-09, 9.5122e-12, 1.0179e-13, 9.4028e-13, 3.5981e-13, 5.2790e-13,\n 1.9207e-11, 3.6138e-13, 1.3392e-12, 3.4916e-12, 5.2623e-11, 2.4523e-13,\n 4.7412e-14, 4.6852e-09, 8.7430e-13, 3.8759e-13, 3.9756e-11, 1.3568e-13,\n 8.4631e-11, 1.8784e-12, 5.4011e-12, 3.4379e-12, 5.5889e-11, 6.6518e-13,\n 7.5240e-11, 2.4954e-11, 9.9763e-14, 2.6982e-10, 7.6900e-13, 7.5200e-10,\n 2.2993e-10, 1.3348e-10, 4.7018e-13, 1.1210e-12, 5.7481e-10, 3.3575e-12,\n 4.1762e-12, 8.2005e-11, 1.5898e-11, 1.2507e-13, 3.4361e-12, 1.4242e-10,\n 9.0232e-13, 8.2874e-14, 3.1658e-12, 2.1070e-11, 1.0935e-11, 5.2193e-10,\n 8.3231e-12, 3.5130e-10, 3.0269e-13, 1.7089e-12, 1.6460e-11, 6.0108e-14,\n 1.4609e-13, 1.2193e-12, 6.1795e-11, 1.1700e-11, 9.2254e-13, 1.1421e-11,\n 6.1056e-12, 6.7575e-11, 2.1501e-11, 2.5367e-13, 1.6284e-13, 1.2077e-11,\n 1.9937e-10, 8.2779e-13, 2.9109e-12, 2.6424e-13, 5.1675e-10, 6.4801e-14,\n 2.4532e-12, 1.8716e-11, 1.7977e-13, 1.0012e-11, 5.1010e-13, 1.7844e-10,\n 2.7051e-11, 3.6794e-12, 1.0700e-11, 1.4599e-11, 8.4874e-11, 2.7138e-11,\n 5.9738e-11, 1.2543e-12, 1.7036e-11, 2.7319e-11, 3.1826e-11, 8.5167e-12,\n 8.9687e-11, 6.3939e-13, 6.9546e-12, 2.1507e-11, 2.5848e-12, 4.7829e-10,\n 3.8110e-13, 8.7304e-11, 9.6351e-10, 3.7268e-12, 1.1004e-12, 1.0295e-12,\n 2.2227e-10, 4.8268e-13, 4.5473e-13, 5.1408e-12], device='cuda:0')" + }, + "33": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.8419e-16, -6.2190e-14, -1.1421e-15, -1.0632e-15, -7.4445e-15,\n 1.2915e-15, -1.8065e-14, 2.3431e-15, 1.6061e-15, -7.5248e-15,\n 8.7567e-16, 3.2623e-16, -1.1439e-15, -8.5501e-15, -1.0641e-14,\n -2.9895e-14, -5.3759e-14, -1.0063e-14, -4.9759e-15, 2.1034e-15,\n 2.3748e-15, -4.4267e-14, 1.5217e-15, 5.9553e-15, -1.3540e-14,\n -1.9116e-14, -1.8601e-14, 3.4909e-15, -1.4356e-14, -2.7655e-14,\n 3.9098e-15, -8.6161e-16, 2.1545e-15, -8.8824e-15, 4.7092e-15,\n -8.5656e-16, -3.8576e-15, 3.0391e-15, 7.2149e-15, -5.9811e-16,\n 1.0504e-14, -4.1363e-15, 3.7170e-16, -1.6067e-14, -3.7381e-14,\n 4.5843e-16, -4.0761e-14, -1.3486e-14, 1.9610e-16, -1.9715e-14,\n -5.6717e-15, -9.2466e-16, 1.0153e-14, -6.3402e-16, 1.3304e-15,\n 2.4204e-15, -1.3666e-14, -1.0174e-15, -3.0482e-15, -8.1971e-15,\n -3.1059e-14, -9.2808e-15, 1.2274e-16, -5.6605e-15, -1.1061e-14,\n -1.4497e-14, 1.6838e-15, 3.7305e-15, -9.8029e-15, 1.0263e-15,\n 3.2527e-15, 4.8547e-15, -1.5316e-14, 1.5845e-14, 1.0011e-15,\n -3.9487e-15, -3.6099e-14, -1.6998e-15, -2.4764e-14, 3.4321e-15,\n -4.8498e-15, -3.0751e-14, 6.7577e-15, -1.2095e-14, 1.1681e-15,\n -6.5725e-15, -1.4063e-14, -2.7172e-14, -7.1908e-15, -1.7043e-15,\n -4.0495e-16, -6.0558e-15, -2.8023e-14, 8.9000e-16, -4.5236e-15,\n -2.6923e-14, -3.7068e-14, -1.5847e-14, 2.6750e-16, 9.6370e-17,\n 2.3570e-15, -1.3605e-15, -2.5087e-14, -2.3756e-14, -1.8317e-14,\n -2.4487e-14, -3.6124e-15, 4.4927e-15, -1.2628e-15, -2.4698e-14,\n 2.1229e-15, 2.0929e-15, -4.2832e-15, -5.1059e-16, 7.3483e-16,\n -2.2424e-15, -1.4487e-15, 7.7559e-15, -2.2787e-14, -8.0262e-15,\n -1.8020e-15, -2.0894e-15, -2.1788e-14, -1.9333e-15, -3.4656e-14,\n 1.4083e-15, -1.6799e-14, -9.9604e-15, -6.0141e-15, -2.2751e-14,\n -1.9188e-15, -8.9118e-16, 2.8908e-15, -1.7402e-15, -7.5125e-15,\n -2.6025e-14, -7.6332e-16, -1.5342e-14, -1.9241e-14, 5.8762e-16,\n -9.6096e-17, -4.2259e-14, 1.6098e-14, 9.5672e-15, -7.2810e-15,\n 2.7955e-15, -1.5437e-14, -1.7331e-15, -4.9879e-16, 9.7466e-16,\n -3.9909e-16, -7.9362e-15, -2.4612e-16, -1.9943e-14, 1.9295e-15,\n 4.9549e-16, 4.6920e-15, -1.6260e-14, 3.6196e-15, -8.2955e-16,\n 5.0874e-16, 3.3414e-15, 2.9287e-15, 7.4827e-15, -1.0091e-14,\n -1.5083e-15, -6.4007e-15, 1.8691e-15, 4.0697e-16, 2.5190e-16,\n 2.1041e-15, 2.0457e-15, 8.3613e-16, -1.7275e-14, -5.6387e-15,\n 1.8434e-15, -1.7851e-14, -2.3829e-15, 3.0855e-15, -1.9783e-14,\n -3.3765e-14, 2.5103e-15, 1.6819e-15, -1.0925e-15, -4.3755e-14,\n -3.5282e-16, -1.7720e-14, -2.8279e-15, -3.9344e-14, -5.2283e-15,\n 1.2854e-15, -1.3608e-14, -4.6964e-15, -1.2221e-15, 4.4323e-15,\n -9.0340e-15, -6.4986e-16, -1.7535e-14, -1.0179e-15, -2.7074e-15,\n -1.3852e-14, -4.9714e-15, -1.3288e-16, -2.2416e-14, 1.7010e-15,\n 1.9427e-15, 3.2160e-15, -2.4210e-14, -4.6121e-16, 7.4370e-16,\n -1.5951e-16, -9.5781e-15, 2.1904e-15, -9.6012e-15, 7.8233e-15,\n -7.0137e-15, -6.5618e-15, 1.4822e-15, -1.8775e-15, -1.2072e-15,\n -6.8963e-16, 1.1168e-14, -1.6630e-14, -9.0151e-16, -5.6598e-16,\n -1.3556e-14, -8.6159e-15, -2.7747e-14, -2.5527e-14, -2.6091e-15,\n -3.4604e-15, -6.8137e-16, 9.9812e-16, -3.2799e-14, 7.2767e-15,\n 1.1774e-15, 9.5496e-17, -7.7078e-16, -3.1299e-15, 8.8894e-15,\n -7.7426e-16, 4.4766e-15, -3.2257e-16, -5.4286e-15, 2.3949e-15,\n -1.2825e-15, -7.7934e-15, -9.5076e-16, -5.2686e-15, 1.0180e-15,\n -2.2679e-14, 2.0621e-15, 1.0554e-15, -1.3198e-14, 1.9640e-15,\n 1.0139e-18, 8.2964e-16, -4.7117e-16, -1.0549e-14, -1.2345e-14,\n -1.0138e-14, -5.8598e-15, -2.1731e-15, 1.3084e-15, 2.7564e-15,\n 3.7801e-15, 1.8219e-15, -2.4164e-14, 6.9937e-16, 3.8466e-15,\n 6.4127e-15, 6.8217e-16, -6.2095e-15, -2.9744e-14, -4.4335e-15,\n -1.4827e-15, -3.8699e-14, -5.4024e-15, 3.4555e-15, 9.0566e-16,\n -1.5597e-14, -7.1691e-15, 2.0947e-15, -1.1122e-14, 8.2754e-15,\n -7.6892e-14, -6.2737e-16, -1.7216e-14, -1.6804e-14, 1.0602e-15,\n -2.5988e-14, -1.2170e-14, 3.4533e-15, 9.2660e-15, -4.9994e-16,\n 9.4931e-16, -3.5571e-14, 3.1428e-15, -2.0660e-15, -3.4877e-15,\n -1.9640e-14, -1.3573e-16, -1.1914e-14, -7.9277e-16, 1.3812e-15,\n -1.3366e-14, -2.4274e-15, -4.5553e-17, -3.5161e-14, -1.4784e-14,\n -9.1120e-15, 6.9508e-15, -4.2925e-14, 1.0584e-14, 1.2333e-15,\n -9.9380e-15, -1.2491e-14, -2.6816e-14, -9.4928e-15, 3.4832e-15,\n 8.1785e-17, 5.3786e-16, -8.9173e-16, 3.0054e-15, -1.0427e-14,\n 1.9301e-15, -6.9017e-15, -1.9453e-14, -1.7918e-15, -2.2237e-14,\n 2.7246e-18, -4.2457e-16, -6.2702e-15, -9.7573e-15, 3.5408e-15,\n -5.4323e-15, 9.0694e-16, -3.0753e-14, -3.1127e-14, -3.6060e-14,\n -1.2935e-14, -9.4631e-16, -1.9955e-14, -1.2585e-14, -2.4094e-14,\n -5.0820e-16, -1.5599e-15, 2.3339e-15, -1.5964e-15, -4.8735e-14,\n 3.7510e-15, -2.8207e-15, -3.5123e-16, -4.7959e-15, 3.3699e-15,\n -1.3967e-14, -3.1942e-14, -5.4287e-17, -8.8086e-15, -2.6132e-15,\n -7.5901e-15, -2.8723e-14, -3.6863e-14, -9.7419e-15, 9.5648e-15,\n 5.0526e-16, 2.0839e-15, -1.0326e-14, -2.1564e-14, -5.6356e-14,\n 3.8685e-17, -1.8142e-15, -4.6736e-16, -2.3540e-14, -2.9149e-14,\n 5.5241e-17, -1.4861e-14, 7.9398e-15, -1.1719e-14, -1.5567e-15,\n -3.7608e-15, 9.0772e-16, -1.1131e-15, -1.9182e-14, -1.3019e-14,\n 8.0187e-16, -1.8002e-14, -8.2610e-16, -9.6218e-15, -1.9765e-14,\n -1.8850e-15, -3.1859e-15, -5.1998e-15, 4.4851e-15, -2.2033e-14,\n -9.0494e-15, -1.8944e-14, -1.4990e-14, -6.9354e-15, -2.7883e-14,\n -2.2210e-14, -1.1184e-14, -5.1424e-14, 1.0404e-14, -1.8736e-14,\n -1.5100e-14, -4.6524e-15, -5.3806e-14, -3.0549e-14, -5.9995e-15,\n -4.1803e-14, 2.0664e-16, -1.8296e-14, -5.3294e-16, -9.1385e-15,\n -2.1934e-14, -4.0731e-15, 5.7625e-16, -4.2336e-14, 4.4577e-16,\n -1.0824e-15, -7.2790e-15, -6.0942e-15, -1.7850e-14, -8.8199e-16,\n 7.3003e-16, -2.0253e-14, 7.3021e-16, 1.1557e-15, 1.1448e-15,\n -1.9380e-14, -1.4833e-14, -1.4254e-14, -1.6261e-15, -1.6607e-14,\n -8.2025e-15, -3.0366e-16, -4.3903e-14, 5.1782e-16, -1.0357e-14,\n -4.0173e-15, 1.7642e-15, -1.5996e-14, -2.7017e-15, -8.7099e-15,\n -4.5488e-16, -6.9046e-15, -1.5301e-14, 3.8693e-16, -2.1650e-14,\n -3.0837e-18, -5.2983e-15, -1.9765e-14, -5.8080e-15, -4.5483e-15,\n -1.0391e-14, 9.0318e-16, -2.2529e-15, -1.5690e-14, -2.1385e-15,\n 1.0533e-15, -4.3980e-15, -1.4989e-14, -3.0149e-14, -9.5611e-15,\n -8.5736e-17, -5.6954e-15, -1.5002e-15, -4.1173e-16, -1.2169e-14,\n -4.4151e-15, -1.8843e-16, -7.8823e-15, 2.2480e-15, 4.1270e-16,\n -1.2796e-16, -2.3521e-15, -9.9944e-16, -5.1436e-16, -2.7747e-15,\n -5.4953e-15, -3.0858e-15, -1.7420e-15, -6.1107e-15, -1.7397e-14,\n -3.5399e-15, 1.0650e-15, 1.0010e-15, -5.0483e-16, 5.1931e-15,\n 1.3651e-14, -2.1789e-14, -4.9049e-15, -4.0279e-15, -1.6260e-14,\n -4.9272e-14, 1.4352e-15, 1.0135e-15, -2.0503e-15, -8.8635e-15,\n -4.9494e-15, -2.5752e-15, 3.8403e-15, -4.9841e-15, -6.3029e-16,\n -2.0037e-15, -3.3171e-14, -1.0558e-15, -2.4351e-15, -3.8047e-15,\n -7.6125e-16, 1.7197e-15, -2.9249e-15, -4.7449e-15, -3.2101e-16,\n -8.0174e-15, -1.9408e-14, -2.0795e-14, -1.6664e-15, -4.3977e-16,\n 3.8173e-15, 1.1079e-14, 2.4080e-15, -8.0148e-15, -1.5968e-15,\n 2.4464e-15, -8.6163e-15, -4.4184e-15, -7.8410e-15, -1.4566e-14,\n -1.1758e-14, 3.2521e-15, -3.2741e-15, -3.6131e-16, -1.4297e-14,\n 1.9297e-15, 7.9933e-16, 5.0517e-15, -1.8306e-14, -4.0743e-14,\n -4.6358e-14, 8.2487e-16, -2.0352e-14, -6.2677e-15, -2.9184e-14,\n 7.2479e-15, -1.4191e-16, 3.5479e-15, 6.4139e-17, 6.3007e-15,\n 2.9221e-15, 2.4092e-15, -2.8602e-15, 1.2545e-15, -9.0535e-16,\n 3.5512e-17, -1.5697e-14, 1.3119e-15, 2.0493e-15, -2.9151e-14,\n -6.4491e-15, -2.5952e-14, 6.0424e-16, -2.9873e-14, -1.8791e-15,\n -7.9548e-15, -4.2321e-15, -1.1927e-14, -1.2813e-14, -2.9580e-15,\n -1.0188e-14, -3.3276e-15, 2.4103e-14, -1.1338e-14, -2.9185e-14,\n -1.1217e-14, -3.7936e-15, -2.5197e-15, -8.9369e-15, -7.9248e-15,\n -2.6285e-14, -2.4819e-15, -5.2060e-14, -2.0234e-15, -2.5982e-14,\n -2.2691e-14, -3.7812e-14, -3.4399e-15, -1.9181e-16, 1.3278e-15,\n 5.9869e-15, 3.0690e-16, -8.6173e-15, -1.2403e-15, -7.0819e-15,\n 4.1788e-15, -6.5698e-16, -7.1774e-15, -6.4654e-15, 3.5297e-16,\n -1.9638e-14, -1.8918e-15, -3.8411e-14, 1.5961e-15, -2.3124e-14,\n -4.7856e-15, -4.5877e-14, 2.1232e-15, 1.8429e-15, -1.8653e-14,\n -1.7815e-16, -5.1299e-16, 2.3187e-15, -2.0501e-14, -3.7916e-15,\n -1.7306e-15, -2.9763e-14, -7.1898e-16, -1.9772e-14, 6.1525e-16,\n 6.5847e-16, -1.8634e-14, -6.1563e-14, -3.5823e-14, -4.8298e-15,\n -2.5748e-15, -1.3059e-14, 1.2122e-15, 3.6801e-15, -4.9663e-15,\n 1.7990e-15, 9.2362e-15, 3.3126e-15, -1.2379e-14, 1.2016e-15,\n -1.5218e-14, -2.0038e-14, -4.0850e-15, -1.5749e-14, -5.7246e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.1043e-11, 1.0379e-11, 2.5311e-12, 3.5535e-12, 3.0919e-13, 2.7692e-13,\n 1.0122e-09, 5.1142e-12, 5.8917e-13, 1.2658e-10, 4.9976e-13, 4.7939e-12,\n 1.4523e-11, 4.6330e-11, 3.3679e-10, 1.3360e-11, 3.0728e-11, 9.5333e-10,\n 3.2691e-11, 2.6637e-11, 1.1943e-12, 2.5310e-12, 2.9957e-13, 9.6491e-11,\n 4.8818e-10, 3.3733e-10, 6.4551e-10, 1.2229e-11, 2.3208e-11, 6.6393e-10,\n 8.6419e-11, 1.1932e-12, 4.9082e-11, 3.9195e-12, 1.9278e-10, 5.1125e-13,\n 2.1776e-11, 1.1994e-12, 1.9261e-11, 1.0565e-10, 9.7654e-12, 1.5879e-10,\n 6.3321e-13, 1.2792e-10, 1.0046e-10, 3.0488e-12, 8.8801e-11, 4.7017e-13,\n 1.7752e-12, 2.9998e-11, 1.9981e-09, 1.0528e-12, 1.4123e-11, 4.4349e-12,\n 7.0097e-13, 3.5125e-12, 8.6577e-11, 3.5505e-11, 3.2006e-11, 5.1337e-11,\n 6.4929e-10, 1.3887e-12, 5.4926e-12, 1.2199e-11, 6.9480e-10, 6.3654e-11,\n 4.0856e-12, 1.4057e-09, 3.6945e-10, 1.2472e-10, 3.2779e-12, 2.7174e-10,\n 1.3278e-10, 2.8856e-11, 2.5746e-13, 7.3449e-13, 1.4352e-10, 5.4360e-13,\n 8.4640e-11, 2.3184e-10, 4.2973e-12, 3.9355e-11, 1.7863e-10, 1.2290e-10,\n 6.9273e-10, 1.3446e-12, 1.4327e-10, 6.5886e-11, 4.1608e-12, 2.5985e-11,\n 3.5982e-13, 5.8268e-13, 3.2062e-11, 3.0074e-12, 1.9365e-12, 9.9189e-11,\n 2.1105e-11, 2.8007e-10, 4.4552e-13, 7.9232e-12, 1.0353e-13, 6.0611e-10,\n 4.0313e-11, 2.9860e-11, 4.1307e-10, 6.3096e-11, 1.6931e-12, 8.7451e-13,\n 1.0461e-11, 7.8472e-11, 2.5788e-12, 9.7194e-13, 1.5844e-10, 1.9968e-11,\n 1.8041e-12, 5.8979e-13, 2.4835e-11, 5.5034e-11, 5.4341e-11, 2.9867e-10,\n 2.0861e-11, 8.2313e-11, 1.4688e-12, 2.6133e-11, 1.4245e-11, 5.3469e-12,\n 3.5217e-10, 3.1673e-12, 2.5521e-13, 2.5437e-10, 1.1097e-12, 5.1360e-11,\n 7.6783e-10, 2.1712e-11, 7.1313e-10, 1.5723e-09, 6.8024e-12, 2.8626e-10,\n 1.0554e-10, 5.7250e-11, 2.2315e-11, 1.9978e-11, 6.6162e-12, 8.3066e-11,\n 1.4282e-10, 9.3192e-12, 3.8606e-10, 6.9855e-12, 2.8503e-12, 1.2269e-10,\n 1.9026e-11, 2.2617e-11, 3.2665e-12, 1.0672e-12, 2.2187e-12, 4.4299e-12,\n 3.8011e-13, 3.7168e-11, 1.9887e-11, 3.0997e-11, 5.9175e-12, 7.6762e-12,\n 3.8397e-12, 2.4481e-10, 8.2103e-11, 1.1616e-11, 2.7067e-13, 1.6781e-12,\n 1.0951e-12, 3.0574e-12, 9.8246e-11, 2.0078e-09, 5.6653e-11, 9.4550e-13,\n 9.3602e-11, 1.0844e-11, 2.7545e-10, 4.8388e-12, 1.1486e-11, 3.5888e-10,\n 5.6710e-12, 4.4545e-12, 2.2756e-10, 6.7550e-13, 1.1932e-10, 8.4946e-12,\n 3.7505e-10, 9.1208e-11, 1.7120e-10, 1.7060e-12, 4.8256e-13, 6.1076e-11,\n 3.9136e-11, 2.7257e-11, 5.1983e-10, 2.5125e-10, 3.1174e-13, 2.6671e-10,\n 3.7656e-12, 1.4924e-10, 9.4347e-12, 9.5569e-11, 2.9041e-10, 8.7935e-11,\n 4.1251e-10, 1.0672e-11, 1.1146e-11, 4.1061e-11, 3.8943e-11, 1.6561e-10,\n 3.8941e-11, 1.8112e-11, 3.6887e-11, 1.4845e-10, 6.2873e-10, 3.4910e-11,\n 4.7598e-13, 1.2733e-10, 2.2088e-10, 1.2441e-11, 2.8795e-12, 4.1965e-11,\n 7.8535e-10, 1.3477e-11, 1.7520e-10, 1.8248e-11, 2.1492e-10, 1.1675e-11,\n 3.7875e-13, 3.4494e-13, 2.9464e-11, 2.7282e-13, 3.0971e-10, 1.3232e-10,\n 9.7561e-11, 5.2730e-12, 1.3612e-11, 3.9202e-12, 5.0803e-11, 5.0800e-10,\n 2.1786e-12, 3.2254e-10, 1.1333e-11, 3.8097e-11, 4.8236e-12, 1.7716e-10,\n 2.7882e-10, 3.0900e-12, 1.9806e-12, 1.2261e-12, 4.1449e-10, 2.3625e-10,\n 5.2955e-12, 6.1646e-10, 2.7775e-12, 3.7479e-12, 1.6426e-12, 2.6577e-13,\n 1.8638e-10, 5.6827e-11, 1.0701e-11, 8.1712e-11, 8.9153e-11, 1.3080e-11,\n 5.6124e-10, 5.9061e-12, 9.1630e-13, 3.1261e-11, 4.5446e-13, 9.6345e-12,\n 5.4604e-13, 2.1600e-11, 5.1292e-11, 9.3873e-12, 3.5869e-10, 6.1372e-12,\n 1.3567e-11, 7.1303e-11, 9.7511e-10, 3.5852e-12, 1.8238e-10, 1.6518e-10,\n 6.7116e-12, 6.9447e-11, 1.3190e-12, 1.6472e-11, 1.3440e-11, 3.4761e-11,\n 1.6558e-11, 3.1407e-10, 1.8032e-10, 1.5077e-10, 5.0525e-13, 4.8692e-11,\n 4.0841e-11, 1.0767e-10, 1.1232e-09, 2.8624e-12, 1.1158e-10, 3.4849e-11,\n 1.1979e-10, 1.9075e-12, 3.3339e-10, 1.7781e-09, 3.7281e-13, 2.0046e-11,\n 1.2716e-10, 4.4941e-12, 2.2157e-12, 1.1504e-11, 1.9412e-11, 6.4577e-10,\n 1.1750e-10, 2.1942e-11, 1.2830e-09, 1.3196e-12, 2.1340e-11, 2.4583e-12,\n 2.8277e-11, 1.3422e-11, 8.3491e-13, 8.4332e-11, 4.1762e-11, 8.7291e-11,\n 1.2622e-11, 7.1305e-13, 3.3791e-10, 4.3522e-11, 1.0351e-10, 4.8273e-10,\n 1.6543e-12, 2.5997e-13, 1.9337e-10, 2.9066e-10, 2.7136e-10, 2.4420e-12,\n 8.4779e-11, 1.1228e-10, 3.2781e-10, 4.0711e-10, 2.5470e-10, 4.1859e-11,\n 3.2747e-11, 5.6274e-10, 7.3245e-10, 6.6697e-13, 1.5116e-11, 4.4457e-13,\n 1.6818e-12, 1.0191e-10, 1.2366e-10, 4.9194e-12, 3.2300e-13, 1.0932e-10,\n 4.3799e-13, 7.1815e-10, 1.8576e-10, 7.1068e-12, 9.8674e-11, 7.7510e-12,\n 5.7571e-12, 1.0815e-09, 1.6263e-10, 7.2179e-11, 4.7527e-10, 3.3215e-11,\n 2.0928e-11, 2.1099e-12, 4.2449e-10, 2.2688e-12, 8.2660e-10, 4.5194e-10,\n 5.6507e-13, 5.2816e-10, 2.9555e-10, 1.3921e-12, 7.0665e-10, 4.1051e-10,\n 5.6186e-10, 6.3472e-12, 5.5426e-10, 6.3450e-13, 2.2289e-10, 4.5367e-12,\n 3.8999e-12, 2.0043e-12, 3.2382e-12, 1.0308e-12, 2.3026e-10, 2.1549e-11,\n 1.1667e-10, 3.4080e-11, 1.3808e-11, 1.6560e-12, 1.5968e-12, 1.5118e-11,\n 2.6322e-11, 1.6645e-10, 2.2354e-10, 4.7406e-12, 1.5877e-11, 1.0472e-10,\n 6.1930e-10, 1.0698e-09, 9.1222e-11, 5.5924e-11, 4.7575e-12, 2.1374e-11,\n 3.7463e-10, 1.4764e-10, 1.2183e-12, 6.4112e-12, 1.3301e-11, 7.2312e-12,\n 1.0354e-09, 5.5122e-10, 1.4295e-10, 5.4738e-12, 2.1571e-11, 2.8643e-12,\n 1.0388e-11, 4.8928e-12, 1.5801e-12, 1.5732e-10, 3.3674e-12, 8.0594e-12,\n 4.0998e-12, 3.2815e-12, 3.0443e-12, 1.8818e-11, 2.1166e-11, 5.8663e-10,\n 7.1640e-11, 2.6257e-12, 3.5481e-11, 1.1029e-11, 1.1130e-10, 6.1369e-11,\n 7.8219e-11, 1.5885e-11, 2.5317e-11, 1.1582e-10, 2.2699e-10, 2.2443e-12,\n 1.8213e-10, 3.2544e-13, 1.1375e-09, 3.8752e-11, 1.9136e-12, 4.2814e-11,\n 1.0371e-12, 9.3525e-14, 3.9118e-12, 4.1975e-13, 1.1495e-11, 1.6798e-10,\n 3.0669e-13, 6.6565e-12, 1.9164e-10, 2.1875e-11, 4.1675e-13, 7.1683e-12,\n 2.5428e-11, 9.7152e-11, 5.9647e-11, 6.8406e-13, 2.1418e-10, 6.9823e-11,\n 2.8413e-13, 1.8022e-12, 2.0370e-12, 6.4772e-12, 4.1849e-11, 6.1227e-13,\n 2.3162e-12, 4.5640e-12, 8.0611e-11, 6.2773e-12, 1.9657e-11, 7.0471e-10,\n 1.1659e-12, 1.2153e-11, 3.2803e-12, 5.3377e-11, 2.7033e-11, 1.0164e-11,\n 2.2002e-12, 2.6502e-12, 3.1646e-12, 2.4498e-12, 4.7469e-10, 7.9735e-10,\n 1.3882e-10, 5.0788e-11, 6.5326e-10, 1.5313e-10, 5.2597e-13, 4.1692e-12,\n 1.4806e-11, 2.7828e-11, 2.0885e-10, 1.1925e-09, 3.2030e-12, 5.9772e-10,\n 2.7589e-12, 5.6790e-11, 1.6856e-10, 5.9146e-10, 8.1287e-11, 1.5491e-12,\n 1.5629e-11, 3.6983e-11, 4.7186e-12, 5.6765e-11, 1.7838e-11, 2.2027e-11,\n 1.8856e-11, 1.3303e-11, 9.2680e-10, 7.7238e-12, 8.1415e-10, 9.2179e-11,\n 4.3875e-13, 5.8015e-11, 6.8791e-12, 3.7569e-12, 1.0010e-10, 8.5638e-11,\n 9.4114e-12, 1.1578e-11, 1.3998e-12, 8.5088e-10, 4.7373e-12, 3.1623e-13,\n 4.9027e-10, 3.6976e-12, 3.1285e-13, 4.0935e-11, 2.7043e-12, 2.1393e-12,\n 1.5217e-10, 8.5720e-13, 1.9734e-11, 1.6204e-11, 1.0158e-10, 2.9104e-13,\n 9.8153e-13, 1.6962e-09, 3.8002e-13, 3.5479e-13, 1.5562e-11, 8.9533e-13,\n 2.6565e-10, 4.4936e-12, 2.1382e-10, 2.7356e-12, 2.1930e-11, 5.5740e-12,\n 1.2518e-10, 1.1529e-10, 9.7005e-13, 6.5681e-10, 3.3318e-13, 6.6238e-10,\n 8.5211e-10, 6.0957e-11, 9.7269e-12, 1.8968e-11, 3.5632e-10, 7.3160e-11,\n 8.1601e-11, 9.1477e-11, 1.1991e-10, 3.4646e-13, 1.9492e-11, 7.7158e-11,\n 1.5235e-11, 1.3934e-12, 1.4976e-11, 5.4251e-10, 1.6015e-11, 3.0499e-10,\n 5.2865e-11, 4.4041e-10, 2.4841e-12, 1.8483e-11, 1.8591e-11, 1.5034e-12,\n 2.9467e-12, 6.7039e-13, 2.7867e-10, 4.2944e-12, 2.7864e-11, 1.0507e-10,\n 1.8273e-11, 1.2937e-10, 2.5559e-11, 6.4262e-12, 3.5395e-12, 3.6313e-11,\n 1.7512e-10, 4.4934e-13, 3.7886e-12, 7.8086e-12, 6.5293e-10, 4.1375e-12,\n 5.2438e-12, 6.9887e-12, 7.9444e-12, 7.1571e-11, 1.3885e-11, 6.8402e-11,\n 3.7794e-11, 5.7645e-11, 1.9530e-10, 1.3816e-11, 3.9805e-11, 5.2055e-11,\n 4.7627e-11, 1.8340e-11, 1.9947e-11, 1.3938e-10, 1.5657e-11, 2.4021e-11,\n 3.0068e-10, 1.4128e-11, 9.8410e-11, 9.3707e-12, 2.0205e-11, 7.5445e-10,\n 4.5128e-12, 3.0753e-10, 4.9725e-10, 4.7773e-11, 1.0975e-12, 1.0075e-11,\n 5.6618e-11, 1.3296e-11, 1.7193e-11, 1.6861e-11], device='cuda:0')" + }, + "34": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-9.9095e-18, 1.0303e-17, -5.6052e-45, ..., 1.1155e-16,\n -1.1831e-15, 5.6052e-45],\n [-1.0397e-16, -1.8123e-16, 5.6052e-45, ..., -2.8561e-16,\n -2.6440e-14, -5.6052e-45],\n [ 6.4966e-17, 2.8721e-17, 5.6052e-45, ..., 3.9237e-16,\n 1.1172e-14, 5.6052e-45],\n ...,\n [ 2.4494e-17, 1.2163e-17, 5.6052e-45, ..., -5.0412e-16,\n 2.6840e-15, 5.6052e-45],\n [ 4.5571e-18, 2.6903e-17, -5.6052e-45, ..., 1.0572e-15,\n -2.6791e-15, 5.6052e-45],\n [-2.5370e-17, -7.3474e-18, 5.6052e-45, ..., -3.0303e-16,\n 2.6289e-15, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.2194e-10, 9.8324e-11, 1.0822e-15, ..., 1.5221e-10, 3.0812e-11,\n 6.2622e-13],\n [2.9821e-12, 2.0166e-12, 4.2952e-15, ..., 2.0324e-12, 8.7854e-13,\n 2.2064e-13],\n [1.9786e-12, 1.6612e-12, 1.7226e-15, ..., 2.2752e-12, 5.2842e-13,\n 6.6339e-13],\n ...,\n [1.9716e-11, 6.8794e-12, 2.0595e-15, ..., 1.5899e-11, 4.8742e-12,\n 4.0053e-14],\n [5.3705e-12, 7.7687e-13, 5.0908e-17, ..., 1.5510e-12, 1.1200e-12,\n 7.6921e-15],\n [1.0050e-10, 5.4829e-11, 7.1846e-15, ..., 1.5850e-10, 3.0345e-11,\n 2.0996e-15]], device='cuda:0')" + }, + "35": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 2.9576e-13, -3.2367e-12, 1.1067e-12, 1.2876e-13, -2.1264e-13,\n -6.2948e-14, -7.8495e-14, 5.6324e-14, -1.3361e-13, 1.2507e-13,\n 4.6527e-13, -1.5824e-13, 3.4199e-13, 1.7838e-14, -5.7450e-13,\n -5.8759e-14, 1.3746e-13, -2.5962e-13, 5.1187e-13, 2.0294e-13,\n -1.4580e-13, -6.3335e-13, 7.3871e-13, -2.4435e-13, -5.0323e-13,\n -5.1305e-13, -3.9968e-13, 2.8425e-14, 2.9459e-13, 1.1542e-13,\n 9.9457e-14, 7.4352e-13, 1.9964e-13, -2.1228e-12, -8.3913e-15,\n 1.4184e-13, 9.8531e-14, 2.4211e-13, -2.3374e-13, 6.0273e-13,\n 1.9699e-13, 8.8227e-13, 1.7131e-13, -3.2873e-13, -6.8135e-13,\n 8.0390e-14, -1.6591e-13, 7.6522e-13, 4.1982e-13, -1.0621e-12,\n -2.6896e-13, 8.1219e-13, 3.9735e-13, 3.6355e-13, 2.3098e-13,\n -2.4437e-15, -1.2687e-12, 1.0547e-13, 1.0208e-13, 4.2669e-14,\n -1.4318e-12, 3.0595e-14, 9.2898e-14, 2.2354e-13, -8.3785e-14,\n 4.9531e-14, -7.5217e-13, 1.5156e-12, -1.9999e-13, 7.1602e-13,\n -7.4139e-13, 6.7180e-14, 4.9679e-13, 2.1136e-13, 2.9314e-13,\n 1.4168e-13, -3.9770e-13, 4.5310e-14, -7.3524e-13, -2.0584e-14,\n 3.6495e-13, 1.1821e-13, 1.5020e-13, 2.5989e-14, 8.2704e-14,\n -3.2585e-13, -4.2638e-13, -9.2851e-13, -1.1310e-13, 7.1355e-14,\n 1.1554e-13, 1.2927e-13, -1.0995e-12, 5.9879e-14, -4.7488e-13,\n -1.2330e-12, -1.2458e-13, -7.4422e-15, -1.6365e-13, 8.1049e-13,\n 3.7460e-14, -8.9672e-15, -8.2206e-14, -3.8533e-13, 6.0443e-14,\n 3.8678e-14, 5.4967e-13, 2.0155e-12, 7.2098e-13, -2.0528e-13,\n -1.8552e-13, 1.0279e-13, -1.7308e-13, -5.9285e-14, 4.8478e-13,\n 2.0406e-13, 1.3863e-13, -1.1246e-13, 1.5420e-13, -3.8873e-13,\n 2.7709e-13, 2.4690e-13, 1.4050e-13, -4.5645e-14, 2.0013e-13,\n 3.4325e-13, -3.7350e-13, -4.8284e-14, -3.2096e-13, -2.1509e-13,\n 1.3960e-13, -2.5967e-13, 2.1109e-13, 2.3167e-13, 9.3590e-14,\n -9.3853e-13, 7.5871e-14, 2.9249e-14, -5.1873e-13, 1.6993e-13,\n -2.1114e-13, -2.1989e-13, 2.6048e-14, 3.9588e-14, -1.7981e-13,\n 2.2763e-13, 1.9787e-13, 2.8360e-13, 4.0469e-13, 2.3373e-13,\n 3.7112e-13, 1.0035e-13, 8.4931e-14, -7.7247e-14, 9.5353e-13,\n 1.3864e-13, -4.3537e-14, -7.7520e-13, 2.0880e-16, 2.2066e-13,\n 7.1625e-13, 1.9366e-13, -8.8265e-13, 2.0893e-13, -1.4417e-13,\n 1.7449e-13, 9.5377e-14, 9.8857e-14, 9.7140e-14, 1.0998e-12,\n -7.2518e-14, 5.3455e-13, 1.0178e-13, -2.7573e-13, -1.1555e-12,\n 8.5202e-14, 2.0456e-14, 4.2675e-13, 5.2170e-13, 6.1963e-14,\n -2.9261e-13, 1.2111e-12, -1.6768e-12, -3.3979e-13, -7.4895e-13,\n 3.5661e-13, -1.4596e-13, 2.2154e-13, -4.8396e-14, 6.6002e-13,\n 3.5497e-13, 3.9131e-14, 5.9056e-14, 6.7776e-13, 8.0548e-13,\n 6.5922e-14, 2.0230e-13, 3.6518e-14, 5.2809e-13, -6.4881e-14,\n -4.2083e-13, 1.5440e-13, 3.5771e-14, -1.2302e-12, 2.2652e-13,\n 1.4149e-12, 1.3147e-13, -2.3814e-13, 9.0342e-13, 6.9923e-13,\n 3.2480e-13, 2.7190e-13, -1.1781e-13, -3.0116e-13, 6.4120e-13,\n 4.4758e-14, 1.9990e-13, 4.3767e-13, 1.3605e-13, 1.2849e-13,\n -1.0577e-13, 4.0477e-13, -4.9895e-13, -1.8373e-13, 1.0031e-13,\n -1.8102e-13, 4.4645e-13, -1.9972e-12, -8.8074e-14, 4.3640e-14,\n -1.4019e-12, 1.4810e-13, 6.0819e-13, -1.0120e-12, 2.5940e-13,\n 1.4014e-12, 1.2524e-13, 9.7378e-13, 6.3973e-14, 3.9463e-13,\n -1.1050e-13, 8.9842e-13, 1.1896e-13, -7.3054e-13, 1.3566e-12,\n 4.1461e-13, 6.5749e-14, 1.1078e-12, -1.6700e-12, -5.4473e-13,\n 8.2076e-14, 2.7003e-13, -1.0279e-13, -4.9920e-13, -2.3072e-13,\n 4.2759e-13, -5.9951e-13, 3.8283e-13, -2.6851e-13, -6.3219e-13,\n 4.6617e-15, -7.3810e-14, 2.5866e-13, 8.1639e-14, 4.3499e-13,\n 1.8563e-13, 1.1734e-12, -1.0291e-13, 7.4649e-13, -4.1217e-14,\n 3.0584e-13, -2.0922e-13, 4.6709e-14, 9.2301e-14, 4.2192e-15,\n -5.6492e-13, -1.2495e-12, -5.0075e-15, 7.2441e-13, -4.3085e-14,\n -6.2314e-14, 1.1685e-12, 4.5479e-13, -1.5358e-12, 3.1192e-13,\n -5.5351e-13, 8.6396e-14, -2.0621e-13, -9.3597e-13, 7.3976e-13,\n -6.3091e-13, -1.1050e-13, -1.3924e-12, 7.5710e-13, 1.8991e-13,\n -9.3650e-13, -2.2579e-13, -2.9281e-14, -8.8350e-16, -2.1802e-13,\n 4.2044e-13, -6.9883e-13, -4.7984e-14, 7.0381e-13, -1.0883e-12,\n 2.4405e-13, 7.1037e-14, 7.3058e-16, -4.5944e-13, 7.1145e-13,\n -5.7600e-13, 7.7128e-13, -1.6962e-12, 2.1533e-12, 2.1103e-13,\n 5.3852e-14, -8.1381e-13, 5.0318e-13, -8.4677e-13, 6.2075e-13,\n 2.0259e-13, 1.9555e-13, 2.6058e-13, 5.4816e-13, -1.9171e-13,\n -2.3361e-12, 1.1939e-13, -4.2633e-14, 2.4678e-13, -9.3375e-13,\n 1.4724e-13, 9.5915e-14, -3.0800e-14, -2.7162e-14, 2.7736e-13,\n 2.9016e-13, 7.6490e-13, -3.9379e-14, -1.0823e-12, -2.6839e-13,\n -4.8742e-14, 4.5565e-13, 3.7033e-14, 2.7925e-13, -5.5969e-13,\n -3.1655e-13, 1.2897e-13, 1.9641e-12, 1.1829e-13, -3.8958e-13,\n 3.2065e-13, 1.4021e-14, 1.4864e-13, 1.0938e-13, -1.6432e-14,\n 3.0462e-14, -9.1657e-13, 1.1101e-12, -1.1019e-13, -2.5198e-13,\n -5.9752e-14, -1.0376e-12, -1.2830e-13, -2.0666e-13, 9.4177e-14,\n 1.7192e-13, 7.0311e-13, 1.0977e-12, -5.3076e-14, -8.0505e-14,\n 3.4164e-13, 2.2576e-13, 2.0744e-13, 4.9182e-13, -6.0077e-13,\n 4.3104e-14, -3.5972e-15, 1.8500e-13, 9.9231e-14, 6.9188e-13,\n 6.6723e-14, -9.5635e-14, 1.4638e-13, 1.7169e-13, -1.5782e-13,\n 4.9560e-13, -6.1683e-13, 4.6811e-13, -2.6349e-13, -1.0903e-13,\n 7.2305e-13, 3.5812e-13, 9.6491e-13, 1.7985e-12, 1.3357e-13,\n 4.1699e-13, -7.3307e-13, -7.2408e-13, 2.8170e-13, -4.2987e-13,\n -6.8270e-14, 1.8430e-13, 8.0500e-14, 2.7086e-13, -4.9816e-13,\n -8.0722e-14, 8.8828e-14, 9.5118e-14, -3.0981e-13, -5.1505e-14,\n -1.1611e-12, -3.9072e-13, -6.6496e-13, 1.7837e-13, 3.7346e-13,\n -1.2284e-12, -5.8588e-14, 7.2622e-13, -7.0580e-13, 1.1089e-13,\n 1.3455e-12, -2.0108e-13, -6.2975e-13, -6.6164e-13, 1.1594e-13,\n 9.5219e-15, -1.0587e-13, 1.2167e-13, 5.5459e-14, -1.1602e-13,\n 1.0667e-13, 2.7945e-13, -5.4173e-14, -1.8725e-14, 1.4553e-13,\n -1.0762e-12, 4.2250e-13, -9.6710e-13, 4.4860e-13, -4.4369e-15,\n 3.5973e-13, 7.4465e-14, 5.4905e-14, 3.1419e-14, -2.1927e-14,\n 1.8939e-13, -2.5969e-14, 2.5211e-13, 5.5346e-13, -4.4835e-13,\n 6.4492e-14, 2.5692e-14, -7.7975e-13, -6.7570e-13, -1.6452e-13,\n 2.6679e-13, 1.7679e-13, 2.7406e-13, 2.5833e-13, -8.3803e-14,\n 1.3763e-13, 2.9471e-13, -8.1638e-14, -7.8928e-13, -1.1174e-12,\n 1.5515e-12, 3.7485e-14, -2.9216e-12, -3.8496e-14, 1.0815e-12,\n 6.7329e-13, 1.2412e-12, 5.5465e-14, 1.5901e-13, 1.1163e-12,\n 1.8304e-15, 1.7605e-12, 4.7216e-13, 2.1016e-13, -1.8835e-13,\n 1.0306e-13, 1.3560e-12, -1.2082e-13, 1.3554e-13, 4.4239e-14,\n 3.3787e-13, 3.1353e-13, 1.1000e-13, -1.0912e-14, -1.2671e-12,\n -5.8624e-14, -1.0835e-13, 1.9816e-13, 1.6662e-13, -4.1390e-13,\n -1.3695e-13, -5.4018e-15, 9.4907e-14, -2.7592e-13, 4.4439e-15,\n 1.0886e-13, 1.5591e-13, 9.7326e-13, 8.6307e-14, 2.8111e-13,\n -5.9114e-13, -2.5733e-13, -2.4877e-13, 3.3837e-13, 1.7582e-13,\n 5.0958e-14, 1.4806e-13, -4.7899e-13, 4.1313e-14, -9.8976e-14,\n 3.1485e-13, -1.6185e-13, -6.4830e-13, -1.4634e-13, 7.5903e-13,\n -3.1631e-13, 1.8263e-13, 2.3025e-14, 1.7780e-13, -7.0128e-13,\n 2.1667e-14, 2.5991e-14, 2.4308e-13, -4.1919e-13, -4.5889e-13,\n -1.9352e-14, 4.6265e-13, -1.2328e-13, 1.8658e-13, 3.4496e-15,\n 9.4282e-14, 1.1004e-12, 5.4864e-13, -1.9559e-13, -1.0738e-12,\n -1.3457e-12, -3.8026e-13, 1.9315e-13, 1.4337e-13, -1.5721e-13,\n -3.2552e-12, 3.1707e-13, -1.4745e-13, -2.7593e-13, 7.6580e-13,\n -1.8253e-13, 7.5197e-13, 9.9181e-14, -2.4866e-13, 6.6518e-14,\n 4.1272e-13, 2.0727e-13, -7.0735e-13, 3.9376e-13, -3.4986e-13,\n 2.4651e-13, 4.3937e-14, 2.8271e-13, 2.1764e-13, 2.4285e-13,\n -7.1836e-14, 1.9102e-13, -1.8236e-12, 6.1817e-13, 4.2477e-14,\n 1.8166e-13, 5.6896e-14, 1.9837e-13, 1.4930e-13, -2.5293e-12,\n -3.6454e-13, 1.8913e-14, 1.9792e-13, -3.8467e-13, -9.0319e-14,\n -6.5481e-15, 1.0136e-13, -8.7788e-13, -2.5418e-13, 1.2973e-13,\n -5.5069e-13, -6.3419e-13, -7.7775e-14, 2.5850e-13, 7.9563e-13,\n 1.5976e-13, 2.2143e-13, -4.2544e-13, -2.5049e-13, 1.5022e-13,\n 4.2855e-14, 7.2352e-14, 4.1402e-14, 6.7630e-14, -2.0243e-13,\n -3.4949e-13, -2.5625e-13, -2.1851e-13, 2.2476e-13, 3.6667e-13,\n 6.9170e-13, -1.5515e-12, 2.9851e-13, -1.9131e-14, -1.5437e-12,\n 4.6834e-13, 1.0423e-13, -6.2261e-14, -8.5936e-13, -1.2501e-13,\n -1.2944e-12, -9.7294e-13, -8.9299e-14, 6.4390e-14, 9.4783e-13,\n 6.5979e-13, 7.3296e-14, -8.3504e-13, -7.1022e-13, 1.7173e-12,\n 7.5083e-13, -2.0204e-13, -5.9525e-13, 1.2807e-13, -7.6256e-13,\n 4.2375e-13, 2.3836e-13, 5.1331e-13, 2.4293e-14, 3.4676e-14,\n -3.5894e-13, -3.5309e-13, 2.5025e-13, 5.1746e-14, 1.4005e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.5576e-06, 5.8933e-08, 4.5820e-08, 1.3168e-08, 1.9173e-08, 1.5085e-06,\n 2.8375e-06, 2.6521e-07, 3.6001e-09, 4.6133e-08, 9.6428e-08, 2.1762e-08,\n 4.8113e-07, 5.6845e-09, 2.0852e-07, 2.3339e-08, 1.4020e-06, 1.2728e-08,\n 8.4281e-07, 2.3414e-09, 7.3510e-09, 1.0281e-07, 2.6662e-07, 9.9418e-09,\n 5.2778e-09, 4.0270e-07, 2.6562e-06, 1.9401e-06, 5.2997e-08, 9.4636e-09,\n 6.4472e-07, 8.1601e-08, 2.3139e-08, 4.9643e-08, 1.8118e-08, 3.0486e-07,\n 2.3934e-08, 2.6880e-09, 6.2628e-07, 4.4537e-09, 5.2265e-08, 8.9745e-07,\n 1.7445e-07, 2.2385e-09, 3.3651e-08, 3.3927e-08, 7.7593e-09, 4.1686e-07,\n 6.6844e-08, 7.9595e-07, 7.3954e-09, 7.5424e-07, 1.8439e-08, 2.3678e-07,\n 6.4275e-07, 2.6418e-06, 3.8287e-06, 1.3399e-08, 1.9814e-06, 1.2565e-08,\n 6.2724e-07, 4.4479e-08, 9.8610e-08, 6.2320e-08, 2.4679e-07, 8.1276e-09,\n 1.6410e-06, 1.2172e-06, 3.0257e-07, 1.4806e-06, 5.0622e-06, 1.5788e-08,\n 2.5680e-08, 3.9672e-09, 2.1986e-07, 6.1292e-08, 1.6199e-07, 1.3688e-08,\n 3.4661e-07, 1.4386e-06, 7.0905e-09, 4.7435e-08, 2.4301e-08, 6.6850e-07,\n 3.0582e-08, 2.1811e-08, 5.2237e-08, 2.8268e-07, 1.9397e-06, 2.5202e-08,\n 2.4156e-08, 4.3441e-08, 3.1224e-07, 1.1498e-06, 1.2233e-06, 3.2122e-06,\n 1.6669e-07, 1.7293e-06, 1.4111e-07, 1.5005e-07, 1.7314e-08, 1.0918e-06,\n 6.6567e-08, 7.0426e-07, 1.4669e-06, 7.2186e-09, 6.6422e-08, 1.0624e-06,\n 3.9481e-09, 9.8050e-08, 1.2125e-06, 1.2039e-07, 2.6404e-08, 6.6374e-09,\n 1.8089e-09, 1.9390e-08, 2.4088e-09, 7.4053e-08, 2.2260e-09, 1.9862e-08,\n 2.1082e-06, 7.8390e-07, 3.1801e-07, 1.0966e-06, 7.4651e-09, 3.1101e-07,\n 4.3448e-09, 8.1578e-09, 4.1535e-09, 8.6548e-08, 6.0638e-09, 7.3069e-08,\n 1.3272e-06, 6.1754e-07, 4.8600e-08, 3.8919e-06, 2.2256e-07, 6.7235e-07,\n 1.3874e-06, 1.6274e-06, 9.1707e-07, 2.4842e-07, 6.6183e-09, 6.8418e-09,\n 1.9503e-08, 1.6909e-08, 6.3208e-08, 2.6693e-06, 1.1457e-06, 1.1941e-08,\n 2.3226e-07, 1.1952e-07, 3.4913e-08, 1.8887e-09, 3.9715e-07, 1.0835e-07,\n 7.8019e-08, 5.2128e-07, 2.0546e-06, 1.6001e-07, 2.4791e-06, 3.9191e-09,\n 1.3547e-06, 2.7904e-09, 5.3928e-07, 1.2080e-08, 2.5094e-09, 3.6583e-08,\n 6.4033e-08, 1.3436e-06, 1.4040e-09, 1.5954e-07, 6.5061e-08, 1.1927e-06,\n 1.0336e-06, 2.4363e-07, 2.0138e-07, 5.1486e-06, 5.0507e-08, 3.0765e-06,\n 2.3421e-09, 5.8433e-08, 1.0095e-06, 4.1804e-08, 4.1012e-07, 2.4590e-09,\n 2.7024e-06, 1.9668e-06, 1.4112e-08, 6.1440e-09, 8.4519e-09, 1.4486e-08,\n 3.6142e-08, 3.7732e-09, 2.2237e-06, 5.9598e-09, 2.2550e-08, 1.7454e-09,\n 3.3137e-06, 1.4930e-06, 1.4057e-06, 1.0610e-07, 2.7269e-08, 5.0283e-08,\n 1.1701e-07, 1.6361e-06, 2.9591e-07, 4.9672e-09, 4.0332e-06, 2.5733e-06,\n 1.2828e-07, 8.3869e-08, 1.3548e-08, 4.6532e-07, 2.4906e-06, 1.9465e-09,\n 2.1639e-08, 6.7307e-09, 5.0315e-07, 6.0492e-08, 1.0407e-07, 5.5400e-09,\n 3.8156e-06, 2.2603e-08, 1.0610e-06, 1.2501e-06, 1.8499e-07, 8.5830e-07,\n 1.4025e-08, 1.2473e-06, 2.8460e-08, 1.0014e-08, 4.7841e-07, 2.3645e-07,\n 2.1450e-06, 3.1233e-06, 1.0540e-07, 2.0266e-07, 7.5913e-09, 8.1960e-08,\n 5.1890e-07, 1.1985e-08, 6.7345e-08, 7.8609e-08, 6.2877e-09, 4.2763e-09,\n 5.6610e-07, 1.7881e-06, 2.4706e-06, 2.5174e-06, 4.9530e-07, 3.2510e-08,\n 1.0441e-08, 2.6069e-08, 2.8762e-07, 1.0833e-06, 1.1008e-06, 7.2948e-09,\n 1.7296e-07, 1.0930e-06, 2.5711e-07, 5.5939e-09, 1.8080e-07, 1.9763e-07,\n 1.4008e-06, 1.4466e-07, 1.3136e-08, 4.9634e-07, 2.3829e-06, 1.7332e-08,\n 1.8075e-07, 4.4137e-09, 5.9680e-07, 4.6951e-07, 2.4113e-08, 8.7948e-09,\n 3.4045e-07, 2.7690e-09, 6.3786e-07, 1.9648e-06, 1.2631e-06, 6.1868e-08,\n 1.2232e-07, 1.4721e-07, 3.0507e-07, 1.2025e-07, 2.3768e-09, 1.4167e-07,\n 5.0047e-07, 5.1732e-09, 8.9328e-08, 1.4639e-08, 3.6942e-08, 5.7314e-09,\n 3.7676e-09, 1.5834e-06, 1.3448e-06, 7.2858e-08, 1.1049e-07, 3.1325e-09,\n 1.1352e-06, 1.3415e-07, 3.3230e-08, 1.0657e-06, 5.6121e-07, 1.7482e-08,\n 1.5414e-07, 3.3483e-08, 1.1436e-06, 2.7980e-07, 4.3334e-07, 1.6131e-06,\n 4.5409e-06, 2.4920e-06, 4.6259e-07, 5.4167e-08, 1.7031e-07, 1.8038e-07,\n 3.3827e-06, 5.4913e-06, 5.2066e-09, 1.8765e-08, 1.5238e-09, 6.2974e-07,\n 1.6806e-06, 2.0177e-07, 3.1966e-07, 1.8315e-09, 5.5567e-09, 2.6394e-06,\n 4.7597e-07, 1.3734e-07, 1.7990e-07, 1.0915e-08, 2.0866e-06, 1.5825e-07,\n 5.6035e-06, 1.9582e-09, 2.8594e-06, 4.9006e-06, 1.2904e-06, 4.5223e-09,\n 2.3077e-07, 3.1080e-09, 3.3634e-07, 1.0160e-06, 1.6054e-07, 2.6519e-06,\n 8.7344e-09, 1.8780e-06, 2.1395e-09, 1.1205e-07, 4.9644e-09, 9.8827e-08,\n 3.4164e-09, 1.1922e-06, 1.4289e-07, 5.9189e-07, 3.6776e-09, 1.6734e-07,\n 3.0410e-08, 7.1709e-07, 1.7454e-08, 1.0929e-08, 3.1559e-08, 1.2263e-07,\n 6.3280e-08, 1.4976e-07, 7.1156e-07, 4.4104e-07, 1.6282e-06, 6.4685e-08,\n 1.5797e-08, 4.8655e-07, 1.2457e-08, 5.1672e-08, 6.2542e-07, 3.2798e-07,\n 9.1703e-08, 1.2352e-06, 5.0647e-08, 8.0129e-09, 9.6281e-09, 1.0432e-08,\n 4.6109e-09, 5.9382e-08, 2.2914e-07, 5.6195e-08, 1.8978e-07, 3.9894e-09,\n 2.8819e-07, 9.9977e-09, 3.5364e-06, 3.1141e-06, 1.2501e-08, 5.4954e-07,\n 1.9484e-07, 1.7519e-07, 7.0620e-08, 2.0628e-06, 1.9495e-09, 6.2536e-08,\n 4.9118e-08, 6.6214e-07, 2.7106e-08, 2.3827e-09, 1.0529e-07, 8.1573e-08,\n 3.6509e-07, 1.9919e-07, 6.1691e-08, 6.7670e-08, 2.4482e-09, 1.2855e-07,\n 1.2139e-07, 1.3179e-06, 2.6822e-06, 8.4949e-08, 7.1901e-07, 8.3962e-09,\n 4.5052e-08, 1.6502e-06, 5.7549e-08, 1.1487e-08, 2.1963e-08, 6.6620e-09,\n 2.0315e-09, 3.4571e-07, 1.6859e-06, 1.5809e-08, 2.7326e-09, 3.7848e-06,\n 1.7712e-06, 4.0696e-09, 3.3295e-07, 3.7955e-07, 9.8818e-08, 1.7828e-07,\n 1.0088e-07, 6.0407e-08, 6.6749e-07, 3.2236e-08, 2.6571e-08, 2.1603e-08,\n 3.8448e-07, 2.2592e-06, 2.5200e-06, 1.1969e-08, 5.6100e-07, 5.6118e-09,\n 2.6935e-08, 9.3765e-09, 7.7997e-08, 9.5848e-09, 2.1409e-08, 3.0572e-07,\n 4.5390e-07, 2.7166e-08, 6.8676e-09, 3.6677e-07, 1.7349e-07, 2.6664e-09,\n 1.6643e-08, 3.6659e-07, 3.8985e-06, 1.4900e-06, 1.7687e-08, 8.8954e-08,\n 5.5369e-09, 1.0418e-07, 7.7693e-07, 1.5161e-06, 3.2729e-08, 3.2420e-08,\n 3.8375e-08, 2.0219e-08, 1.1168e-08, 2.2799e-07, 1.2622e-07, 2.2424e-07,\n 5.2049e-08, 3.6933e-07, 1.3247e-08, 9.8817e-07, 4.1299e-09, 3.1578e-06,\n 2.6589e-06, 2.0245e-07, 2.9845e-07, 1.6016e-07, 1.4630e-08, 3.3523e-07,\n 1.3018e-08, 5.0278e-08, 8.2550e-07, 1.9768e-09, 1.3868e-08, 1.7423e-06,\n 9.4353e-07, 4.2915e-09, 1.2394e-06, 1.9247e-07, 8.8439e-08, 6.2325e-08,\n 8.2847e-08, 4.0391e-07, 8.8096e-07, 3.8124e-07, 4.2853e-07, 2.8239e-09,\n 9.9088e-09, 8.8625e-07, 3.2166e-09, 5.8608e-09, 9.6322e-08, 1.5019e-06,\n 4.2621e-07, 2.3898e-08, 2.5704e-06, 3.0848e-07, 2.4830e-07, 1.0991e-07,\n 2.7632e-08, 1.1295e-08, 3.6609e-08, 1.9705e-08, 5.9981e-07, 1.2322e-07,\n 6.7407e-07, 1.2564e-08, 1.0198e-06, 8.1766e-06, 2.2366e-07, 2.5618e-08,\n 6.8563e-07, 2.6963e-06, 2.5709e-06, 7.9533e-07, 1.3719e-09, 2.9194e-08,\n 5.0360e-08, 1.2029e-08, 5.5432e-07, 7.3955e-09, 3.3993e-08, 3.8708e-08,\n 1.4976e-07, 1.7621e-08, 2.6345e-08, 3.5809e-09, 3.0333e-06, 9.3758e-07,\n 1.9600e-06, 5.3359e-08, 4.7551e-08, 3.5500e-06, 7.4352e-07, 6.0769e-09,\n 5.5793e-07, 4.3153e-08, 6.2109e-09, 3.0775e-06, 4.2340e-07, 3.9651e-09,\n 3.0401e-06, 1.5796e-08, 5.3375e-09, 2.7937e-06, 3.2008e-09, 1.1738e-08,\n 3.5298e-09, 2.7374e-09, 4.5747e-09, 2.2694e-08, 4.8861e-06, 1.5701e-08,\n 4.1901e-09, 1.2150e-07, 1.5897e-07, 5.3620e-08, 8.0749e-07, 1.5354e-09,\n 3.0136e-06, 2.4589e-08, 5.9976e-08, 2.7962e-07, 3.1052e-07, 5.4725e-08,\n 6.0213e-07, 2.5097e-06, 5.6470e-08, 1.8555e-08, 8.6897e-08, 2.6454e-09,\n 4.2253e-08, 2.0502e-09, 2.9180e-09, 3.3599e-07, 4.0541e-07, 2.8027e-09,\n 1.5398e-07, 2.1749e-08, 1.4574e-06, 5.9166e-08, 2.7621e-06, 6.4646e-08,\n 1.4703e-07, 1.9188e-08, 6.1891e-07, 3.2830e-06, 6.7595e-08, 1.8039e-07,\n 1.2011e-08, 7.7403e-07, 7.9334e-07, 6.6931e-08, 6.1719e-07, 1.7400e-07,\n 2.2178e-09, 3.0710e-06, 1.3466e-08, 1.5181e-08, 7.8834e-08, 5.3416e-08,\n 1.2124e-06, 7.3596e-06, 3.0833e-06, 1.9162e-08, 4.2671e-06, 2.9990e-06,\n 7.2169e-07, 2.5030e-08, 6.1291e-09, 1.5734e-07, 2.1058e-08, 1.3627e-07,\n 6.8463e-09, 3.7834e-07, 4.1156e-08, 3.5402e-06], device='cuda:0')" + }, + "36": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 1.2713e-16, -9.2008e-14, 1.3434e-14, 8.3878e-17, -1.0115e-16,\n -3.3856e-15, -7.0717e-15, 2.7548e-16, 2.4797e-15, -2.4925e-15,\n 4.9739e-15, 3.8377e-15, 3.8909e-15, -2.8034e-16, -1.8402e-14,\n 1.8984e-16, -3.3517e-15, -4.5414e-16, -1.7843e-15, -8.8208e-15,\n 3.6146e-15, -5.2560e-14, 4.3051e-15, 7.9220e-15, 2.2312e-16,\n -9.8002e-15, -3.3649e-14, -3.4495e-15, 8.0427e-16, -2.4147e-16,\n -2.8210e-15, 9.5457e-15, 2.6280e-16, -1.3884e-13, 1.0784e-15,\n 2.4957e-16, 1.0404e-17, -6.0235e-16, -2.1539e-14, 7.6783e-16,\n 1.2424e-16, 8.0305e-15, -7.9315e-15, -3.7599e-16, -1.1648e-13,\n 9.2650e-16, -1.2682e-15, 1.3637e-14, 1.3402e-15, -4.9448e-15,\n -2.1510e-16, 8.0285e-15, -1.4919e-15, 3.3538e-15, 1.2615e-15,\n -5.4444e-15, -4.4946e-14, 5.0053e-16, -7.0154e-15, -1.3190e-15,\n -3.8175e-14, 1.6113e-16, 6.3202e-17, 7.3124e-16, -1.9901e-18,\n -3.8049e-16, -2.5021e-14, 3.0734e-14, -7.1602e-15, 7.0108e-15,\n -7.7470e-15, 1.0906e-15, 5.3551e-15, -2.0241e-15, -1.6975e-16,\n 7.1948e-16, -8.0808e-15, 1.1462e-16, -1.0134e-14, -8.8762e-15,\n -1.4811e-15, -1.5704e-14, 9.6573e-17, 1.0907e-15, -4.2004e-15,\n 6.2607e-16, -9.0345e-15, -1.7258e-14, -2.7868e-15, -1.2444e-16,\n -2.5681e-16, -1.1709e-15, -3.1575e-14, -1.9468e-15, -1.8112e-14,\n -1.7036e-14, 3.6524e-16, 1.1492e-15, -2.9835e-15, 4.1724e-15,\n 4.8129e-16, -9.1522e-16, 1.4264e-15, -1.9996e-14, -2.8848e-15,\n -3.3990e-15, 7.9659e-15, 5.0689e-14, 2.1137e-15, -1.0785e-14,\n -5.1891e-15, 6.3126e-16, 6.5125e-16, 2.0480e-15, 3.1002e-15,\n -1.4020e-15, -7.2437e-18, 7.7697e-16, -1.3023e-15, 4.0999e-15,\n 3.2033e-15, 6.4338e-16, -4.3461e-15, -4.8798e-15, 2.5922e-16,\n 4.3902e-15, -5.4349e-16, -1.0831e-15, -3.5110e-14, -1.0073e-15,\n 6.0030e-16, -2.2404e-14, 1.2213e-15, 1.9529e-15, -5.7578e-16,\n -2.0539e-14, -3.3668e-15, -1.1481e-15, -1.7651e-14, -2.3777e-15,\n -6.5829e-15, -2.8410e-14, 2.4678e-15, -2.3529e-16, -5.4955e-15,\n -1.8811e-15, 1.5793e-15, 3.1542e-15, 1.8815e-15, 5.6439e-17,\n -4.1967e-15, -6.8356e-16, 4.2251e-16, 4.4761e-16, 1.1844e-14,\n -1.1156e-16, 2.6334e-16, -1.2436e-14, -7.7486e-15, -3.9544e-15,\n 1.0525e-14, 1.1561e-16, -1.8827e-14, -4.5056e-16, 1.8753e-17,\n -1.2227e-15, -4.7760e-15, -7.9544e-16, 3.2945e-16, 1.7105e-14,\n 3.8112e-15, 2.6300e-15, 4.2569e-17, -7.6906e-16, -3.2643e-14,\n -7.3882e-17, -8.3491e-16, 2.3011e-15, 1.8083e-15, -5.9180e-15,\n -1.1155e-15, 2.2489e-14, -4.1610e-14, -2.6612e-15, -6.1313e-14,\n -4.5425e-15, -9.5500e-15, -1.9705e-15, -4.4045e-15, -2.8407e-15,\n 8.5936e-16, -2.9312e-15, 6.3674e-16, 2.3399e-15, -1.7046e-15,\n 3.0026e-15, -1.3663e-15, 4.0159e-16, 2.1867e-15, -2.0820e-14,\n -1.2893e-14, 1.2066e-16, 4.1807e-16, -7.9809e-14, -7.2064e-16,\n 1.8743e-14, -8.5387e-15, -3.9942e-15, 1.0724e-14, 8.9957e-15,\n -2.8371e-16, -3.8902e-16, 5.2890e-16, -8.4335e-15, 6.8059e-15,\n 5.4658e-16, -2.5930e-16, 7.7819e-16, 2.9908e-15, -7.4765e-16,\n -1.9350e-15, -1.5145e-15, -1.0083e-14, -1.4342e-15, -3.7022e-15,\n -1.9437e-15, -2.9669e-15, -2.0903e-14, -6.3259e-15, -1.7917e-15,\n -3.6294e-14, -6.8410e-16, 1.6926e-14, -2.6446e-14, 8.2542e-16,\n 2.3835e-14, -9.0907e-16, 2.7849e-16, 2.0297e-15, -1.6702e-15,\n 1.4067e-15, 3.4740e-15, 3.4130e-16, -2.0017e-14, 2.6334e-14,\n -1.5713e-15, -7.9157e-15, 3.6802e-15, -2.7389e-14, -2.0282e-14,\n -9.0727e-16, 2.1081e-16, 6.7472e-15, -2.3777e-14, -1.8006e-14,\n 4.1856e-15, -1.3201e-15, 4.5820e-16, 1.2661e-16, -1.9863e-14,\n -1.0802e-14, -3.7148e-15, -2.5531e-16, -2.6547e-15, 1.6452e-15,\n -3.1301e-14, 3.9771e-14, -2.2079e-15, 2.3730e-15, -2.8938e-15,\n 4.6915e-16, -1.6127e-15, -8.5117e-16, 1.5877e-15, -6.7395e-16,\n -2.9189e-14, -1.4535e-14, 1.2596e-15, 1.5372e-14, -2.1223e-15,\n -1.0055e-14, 1.1797e-14, 6.0855e-16, -1.0037e-14, -2.4269e-17,\n 7.5803e-15, 1.6547e-16, -1.5735e-15, -7.1436e-15, 8.3016e-15,\n -2.7479e-14, -6.7960e-15, -1.1693e-14, 2.4109e-15, 5.0528e-16,\n -2.5831e-14, -1.4589e-14, 2.3624e-15, 9.7981e-16, 5.3195e-15,\n 2.8483e-15, -2.2831e-14, -2.2027e-15, 7.8584e-15, -5.1912e-17,\n -1.1184e-15, -7.0202e-15, -5.6524e-16, -2.1969e-14, 4.7028e-15,\n -1.8511e-14, 1.1481e-14, -3.3327e-14, 4.4430e-14, 4.3452e-16,\n -7.7640e-16, -2.7339e-14, -1.7221e-16, -4.9749e-14, -1.4553e-16,\n -1.4580e-15, -3.6600e-17, 7.7111e-16, 6.5765e-15, -5.3918e-15,\n -1.3716e-13, -8.7384e-16, -4.9852e-16, 2.0722e-16, -1.9090e-14,\n 1.2113e-15, -7.3640e-16, -2.6717e-15, -3.3063e-15, -7.4707e-16,\n -2.0247e-15, 1.5334e-14, 9.3999e-16, -2.7408e-14, -1.4825e-14,\n -9.7706e-15, 1.6343e-15, -2.6657e-14, 1.5448e-16, 2.6389e-15,\n 3.1684e-16, 3.8628e-16, 4.5100e-14, -9.7455e-16, 6.4314e-16,\n -3.5220e-15, 1.7066e-15, 1.8698e-15, 3.3061e-15, 1.2034e-15,\n -5.4111e-15, -1.0804e-14, 1.6135e-14, -2.3818e-15, -9.5290e-15,\n 1.1128e-15, -8.3628e-15, -4.4547e-15, 2.2340e-15, -2.4936e-17,\n 5.7179e-17, 1.1268e-14, 1.9850e-14, 3.6911e-15, 6.2520e-16,\n 8.5722e-16, 2.0745e-16, 2.8561e-17, 1.2490e-14, -1.7324e-14,\n 1.5676e-15, 4.6354e-15, -3.5364e-17, -8.6425e-16, 3.0625e-15,\n -1.1327e-15, 3.9747e-15, -3.7259e-16, -9.9129e-15, -8.5244e-15,\n 1.9792e-15, -1.3241e-14, -1.9456e-15, 1.2494e-15, -1.5986e-15,\n 6.4668e-15, 6.2036e-16, 1.4926e-14, 1.6436e-14, -1.1327e-15,\n -1.9563e-15, 1.2227e-15, -1.1106e-14, -8.8341e-15, -1.8488e-15,\n 3.6915e-15, -2.1830e-15, 3.9159e-16, -2.5843e-16, -3.5508e-15,\n 2.4704e-16, -1.9430e-16, -3.1030e-14, -7.8309e-15, -5.9825e-16,\n -3.8073e-14, -1.1149e-14, 1.3572e-14, -4.3978e-15, -1.4371e-14,\n -2.2205e-14, -2.3563e-15, 8.7754e-15, -2.9590e-14, -8.2552e-15,\n 1.7805e-14, 3.9834e-15, -1.2485e-14, -6.7549e-14, -1.0060e-16,\n 7.5872e-16, 2.4624e-15, -7.9384e-16, -6.8660e-16, 1.9202e-15,\n -1.0988e-15, -7.0924e-16, -8.4610e-15, -2.6315e-16, 7.2117e-16,\n -8.7097e-15, 1.1000e-15, -3.0319e-15, 8.6074e-16, 1.4492e-15,\n 4.7706e-16, 5.5641e-16, -2.8506e-15, 5.3576e-16, -9.6190e-16,\n -8.4342e-16, -8.4605e-15, -1.2121e-15, 4.7088e-15, -1.2705e-16,\n -4.2529e-15, -1.1996e-15, -2.6626e-14, -1.1436e-14, 6.0606e-15,\n 1.5555e-15, -5.3947e-16, 4.5199e-15, -3.1029e-15, 2.4618e-15,\n -4.8142e-17, 1.7972e-16, 3.8083e-16, -2.1495e-14, -3.0361e-14,\n 3.2717e-14, -2.3889e-16, -1.2941e-13, -2.0439e-16, 1.8679e-14,\n 6.4787e-15, 1.9734e-14, -2.8059e-14, -2.4357e-15, 6.9250e-15,\n -5.8046e-15, 4.9201e-14, 6.8559e-16, -6.0000e-16, -1.0241e-14,\n 5.6777e-16, 1.4691e-14, 2.5111e-15, 1.5574e-15, -1.2577e-15,\n -1.0719e-14, -2.0716e-15, 1.0505e-15, -4.3063e-15, -4.7696e-15,\n 3.1279e-15, -3.7058e-15, -3.0754e-16, -1.4542e-15, -9.9987e-15,\n 7.0809e-16, -2.4565e-15, -1.3897e-15, -9.3875e-15, -6.8669e-16,\n -5.9967e-16, 5.6370e-17, 1.2497e-14, 3.3474e-16, -8.1974e-17,\n -3.8893e-15, -3.2521e-15, -3.6808e-16, -9.7003e-15, -8.5427e-16,\n 1.0679e-15, -9.4834e-16, 5.5329e-16, 2.0136e-16, -1.4224e-16,\n 4.8700e-15, -1.1620e-14, -4.0585e-14, -2.2986e-15, 5.2939e-15,\n -4.9780e-15, -1.2670e-15, 3.4543e-16, -3.5973e-16, -4.1037e-15,\n -2.9054e-15, 8.9860e-16, 4.1491e-16, -4.9485e-14, -1.6687e-14,\n -3.8466e-15, -6.1991e-15, -1.1612e-14, -1.0449e-15, -6.1433e-15,\n -1.0644e-14, 3.2280e-15, -1.9510e-15, -4.2394e-16, -3.6480e-14,\n -8.5706e-14, -4.1661e-16, -2.4276e-15, -4.4691e-16, -7.4230e-14,\n -1.2483e-13, 7.0960e-16, 4.2579e-15, 1.7565e-15, 5.5524e-16,\n -5.5456e-16, -1.7860e-15, -7.8999e-15, -1.8478e-14, -1.7184e-16,\n -1.1496e-15, -7.1109e-15, -7.0310e-14, 3.9245e-16, -3.8829e-15,\n -1.7939e-15, -8.6555e-16, 1.9108e-15, -6.6114e-15, -2.2525e-16,\n 1.3152e-15, -4.5240e-17, -1.5310e-14, -6.9288e-16, 1.0889e-15,\n 7.6392e-17, -1.4671e-15, 3.6356e-15, -1.6463e-15, -3.3788e-14,\n -1.1315e-15, -3.2169e-15, -2.4238e-15, -1.3325e-14, -2.0140e-15,\n 4.9643e-15, 3.3890e-16, -8.8564e-15, -5.7431e-15, -6.3567e-14,\n -1.6230e-14, -2.6274e-15, -1.7641e-15, 1.2784e-15, 1.4038e-14,\n -5.0139e-16, -6.8991e-16, -2.8277e-14, 2.1340e-15, -3.2352e-16,\n -1.3671e-15, -2.9548e-16, -2.4721e-15, -1.5404e-14, 1.4564e-15,\n -4.7081e-15, 1.3489e-15, -2.1577e-14, -1.4212e-15, 1.3074e-14,\n 3.7143e-15, -1.9317e-14, -1.8123e-15, -2.4343e-15, -2.1231e-14,\n -8.0767e-16, 6.8415e-16, -5.1255e-16, -2.6166e-14, -3.9733e-16,\n -6.6597e-14, -2.9771e-14, -2.6083e-15, -7.5819e-16, 2.1282e-15,\n 4.2907e-15, -1.1493e-14, -5.7514e-14, -1.0119e-13, 3.5442e-14,\n 1.2399e-14, -7.0646e-15, -4.9119e-15, -4.3538e-15, -4.9065e-15,\n -1.2214e-15, -5.7938e-16, 1.4751e-15, -2.5970e-17, -1.1489e-15,\n -9.7181e-15, -4.7333e-16, -5.8707e-16, -1.3706e-14, -1.7019e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.8853e-10, 5.0443e-12, 3.0529e-11, 3.5511e-12, 9.3888e-13, 1.3964e-10,\n 4.0710e-10, 1.3733e-12, 1.2740e-12, 5.1393e-12, 3.5187e-12, 5.3091e-12,\n 1.2565e-11, 6.3974e-14, 1.3680e-11, 1.2623e-11, 3.8280e-11, 8.4707e-12,\n 8.1134e-11, 1.1701e-13, 2.3148e-12, 8.5668e-12, 1.8046e-10, 3.6115e-12,\n 4.8501e-13, 1.2478e-11, 2.7415e-10, 3.4967e-10, 3.7663e-13, 4.2471e-12,\n 1.3540e-11, 1.1329e-12, 6.0254e-12, 1.7807e-12, 3.8887e-12, 1.4704e-10,\n 7.1130e-13, 4.4927e-13, 4.6457e-10, 1.9201e-13, 7.6767e-13, 3.5165e-10,\n 9.2048e-11, 2.9993e-13, 5.1631e-12, 1.3920e-12, 3.8753e-12, 1.0096e-10,\n 3.6034e-11, 5.7817e-11, 2.3680e-12, 6.8738e-12, 9.5581e-14, 1.0131e-11,\n 2.9507e-11, 7.1106e-10, 1.2277e-09, 2.3468e-13, 6.7648e-10, 1.5898e-13,\n 9.3706e-11, 7.6067e-12, 2.0486e-12, 2.9304e-11, 4.9888e-12, 2.6548e-13,\n 2.3614e-10, 1.3610e-11, 6.4180e-11, 6.3210e-11, 1.7612e-09, 3.4738e-13,\n 1.9876e-13, 5.3943e-13, 2.8156e-11, 2.3232e-12, 2.5247e-12, 3.2890e-12,\n 7.3736e-11, 5.4695e-10, 7.1154e-14, 3.5518e-12, 4.9761e-13, 4.7604e-11,\n 1.9399e-13, 3.4839e-12, 5.9012e-12, 5.1980e-12, 2.1065e-10, 5.6486e-13,\n 1.1340e-12, 4.8370e-12, 1.1259e-12, 1.9761e-11, 5.5195e-10, 7.6467e-10,\n 5.2982e-13, 6.9624e-11, 2.0071e-11, 1.3713e-12, 8.7850e-14, 4.0773e-11,\n 4.6770e-13, 2.2529e-10, 2.8170e-10, 3.1094e-12, 4.7213e-11, 1.2797e-10,\n 8.0500e-13, 7.5154e-13, 1.2837e-11, 1.3821e-12, 8.2656e-12, 1.6171e-13,\n 4.2869e-13, 2.2887e-13, 1.0237e-13, 3.1656e-11, 2.7309e-13, 7.0341e-14,\n 2.9052e-10, 2.1761e-11, 2.4164e-11, 3.5163e-10, 2.1684e-12, 1.8401e-10,\n 5.4774e-13, 3.4507e-12, 3.6640e-13, 5.0819e-12, 4.8320e-13, 6.6060e-13,\n 2.5727e-11, 1.6999e-10, 2.7854e-12, 1.2663e-09, 1.1065e-12, 6.3994e-11,\n 1.0504e-10, 5.6251e-11, 2.1689e-10, 8.4776e-11, 1.2389e-12, 1.5169e-12,\n 1.0266e-13, 1.7235e-12, 8.4929e-12, 7.9883e-11, 2.1419e-10, 2.4873e-12,\n 3.0449e-11, 1.1122e-11, 8.2357e-12, 3.5017e-13, 6.3740e-12, 7.4751e-13,\n 6.5687e-12, 1.9200e-10, 3.5405e-10, 1.0438e-11, 1.2990e-10, 2.3565e-13,\n 2.7386e-10, 7.4927e-13, 2.5913e-12, 4.2985e-12, 4.3181e-14, 1.3900e-11,\n 5.2616e-13, 3.0108e-11, 1.1326e-13, 3.2925e-12, 5.1560e-13, 2.0394e-10,\n 1.5746e-10, 5.9624e-12, 1.3603e-11, 2.5947e-09, 1.8614e-11, 4.8695e-10,\n 6.5749e-13, 1.4120e-11, 8.3227e-11, 8.5648e-13, 9.9738e-11, 5.6330e-13,\n 1.4115e-09, 4.6439e-10, 6.8387e-12, 1.4366e-13, 1.8628e-12, 6.7772e-12,\n 1.3459e-11, 8.9881e-14, 4.6019e-11, 2.1999e-12, 5.8467e-13, 2.2919e-14,\n 4.2497e-10, 5.9935e-10, 3.7613e-10, 3.4529e-13, 3.6363e-12, 1.5457e-11,\n 8.2751e-11, 1.1159e-10, 2.9120e-12, 1.2417e-12, 6.8640e-10, 5.7014e-11,\n 9.0989e-12, 2.7615e-12, 7.3505e-13, 9.8344e-11, 6.7100e-10, 5.7812e-14,\n 6.4628e-13, 4.5912e-14, 9.9470e-11, 1.8212e-11, 8.2247e-12, 1.0882e-12,\n 8.1894e-10, 9.4012e-13, 1.3083e-10, 2.8078e-10, 2.6287e-12, 9.3650e-11,\n 6.4727e-12, 3.0241e-10, 9.6978e-13, 1.0192e-13, 4.8547e-12, 1.9899e-12,\n 6.1508e-11, 1.4994e-10, 1.3787e-11, 1.3572e-12, 1.0218e-13, 6.4143e-13,\n 1.9189e-10, 6.7955e-14, 1.5190e-12, 3.7752e-12, 1.5086e-13, 1.1413e-13,\n 3.1897e-10, 3.5079e-10, 1.0990e-09, 7.1192e-10, 8.6459e-11, 5.2211e-13,\n 3.0167e-13, 1.2486e-12, 1.3663e-11, 6.5252e-11, 1.7333e-10, 1.3814e-13,\n 2.2787e-11, 8.0977e-11, 6.3777e-11, 1.8454e-12, 1.3836e-10, 2.8322e-11,\n 1.3283e-10, 1.9849e-11, 1.7328e-13, 7.7141e-11, 6.9336e-11, 7.6436e-12,\n 7.7327e-11, 2.1227e-13, 2.8467e-11, 7.8829e-12, 6.7072e-12, 3.7036e-13,\n 8.9305e-12, 3.2633e-13, 2.3290e-11, 1.6438e-10, 2.4886e-11, 6.0040e-13,\n 3.7241e-12, 1.6274e-12, 1.1904e-11, 3.9934e-12, 6.2916e-14, 1.6368e-12,\n 3.1204e-11, 1.7197e-13, 2.4832e-12, 7.4102e-12, 2.9741e-13, 1.6408e-13,\n 7.7276e-13, 4.6333e-10, 2.0014e-11, 4.4950e-13, 2.1908e-11, 1.2685e-12,\n 5.7122e-10, 3.5181e-12, 1.5494e-11, 1.8215e-11, 1.2160e-10, 5.5227e-13,\n 6.9029e-13, 9.6493e-12, 7.2916e-10, 4.9981e-11, 1.2876e-10, 9.8258e-11,\n 6.7599e-10, 4.5723e-10, 3.3529e-12, 4.5133e-12, 7.9344e-11, 4.2155e-12,\n 2.3164e-09, 1.0972e-09, 2.6930e-13, 3.7467e-13, 2.9340e-13, 2.7718e-11,\n 2.1888e-10, 6.1486e-12, 1.6667e-11, 6.3197e-14, 2.7544e-13, 1.6042e-10,\n 1.6794e-10, 3.2536e-12, 1.9989e-12, 2.7340e-13, 3.1272e-10, 2.5097e-11,\n 3.1765e-09, 1.5112e-13, 1.0135e-09, 6.2209e-10, 2.5362e-11, 7.3141e-13,\n 1.7046e-10, 1.5910e-12, 2.7885e-10, 2.1677e-11, 1.9121e-11, 3.9447e-10,\n 1.4758e-12, 1.4042e-10, 1.1235e-14, 5.0231e-11, 1.2723e-13, 7.5672e-13,\n 3.0153e-13, 3.8230e-10, 1.2363e-12, 9.6365e-12, 1.5351e-12, 1.0380e-12,\n 1.6912e-12, 7.9625e-11, 3.5582e-12, 5.0346e-12, 1.0067e-11, 6.3693e-12,\n 4.9348e-12, 1.0971e-10, 3.9099e-12, 5.3427e-12, 7.1704e-11, 2.6282e-11,\n 1.4899e-13, 3.5811e-12, 1.3109e-13, 3.1914e-12, 2.3949e-11, 1.1777e-10,\n 4.3047e-13, 2.0210e-11, 5.4194e-13, 1.3820e-12, 2.7609e-13, 1.7806e-13,\n 8.7758e-14, 1.4763e-12, 1.6509e-11, 3.2138e-12, 7.4085e-13, 8.7591e-13,\n 8.6462e-11, 1.8077e-13, 6.0827e-10, 3.4777e-10, 2.8073e-13, 3.2448e-10,\n 4.1886e-11, 3.1802e-12, 4.0454e-11, 6.4625e-11, 1.0128e-13, 3.2641e-12,\n 2.3716e-12, 1.9840e-12, 6.4745e-13, 5.5094e-14, 8.2273e-12, 1.2434e-11,\n 9.5358e-12, 7.2271e-11, 8.9033e-12, 4.7415e-11, 3.7295e-14, 8.2854e-11,\n 2.2114e-11, 3.6678e-10, 2.3082e-10, 5.7602e-13, 1.5307e-11, 3.7640e-14,\n 3.7372e-12, 7.9608e-10, 6.8525e-13, 1.2188e-12, 6.0608e-12, 1.3738e-13,\n 3.6707e-14, 3.4134e-12, 9.4028e-11, 8.8522e-13, 1.4792e-13, 2.2786e-10,\n 6.7101e-10, 1.3809e-13, 2.2375e-11, 6.7017e-12, 2.0551e-12, 5.4829e-11,\n 8.9405e-13, 9.6273e-13, 1.9445e-10, 1.8203e-11, 8.8304e-13, 5.9060e-12,\n 3.8912e-11, 4.6892e-10, 1.0596e-10, 7.8816e-13, 9.3568e-11, 4.0152e-13,\n 1.7723e-13, 1.9671e-13, 6.3777e-11, 6.1670e-14, 8.7155e-12, 5.5804e-11,\n 6.4452e-11, 2.6368e-13, 2.2486e-12, 2.4194e-12, 2.1924e-12, 8.8804e-13,\n 4.5673e-12, 1.1607e-11, 1.4417e-09, 9.5655e-10, 7.7114e-13, 1.6727e-11,\n 3.1299e-13, 1.3604e-11, 7.0574e-11, 7.0749e-11, 1.0919e-12, 1.0020e-13,\n 7.0962e-13, 9.6323e-12, 6.1000e-13, 4.9147e-11, 5.4927e-13, 3.0204e-12,\n 6.3047e-13, 8.8269e-12, 7.8283e-12, 3.3466e-10, 4.1055e-13, 1.2122e-09,\n 1.5211e-09, 3.0069e-12, 5.2770e-11, 6.6031e-13, 3.8739e-12, 1.4342e-12,\n 2.4318e-13, 2.3575e-12, 2.6775e-10, 1.0845e-12, 5.3749e-13, 2.1861e-10,\n 2.4671e-10, 1.3734e-12, 2.1151e-10, 7.7646e-13, 4.6640e-12, 6.8815e-13,\n 1.5074e-12, 7.7768e-12, 2.0113e-10, 2.5592e-12, 4.4186e-10, 2.0680e-13,\n 2.0704e-13, 1.7473e-10, 2.7919e-13, 1.5421e-13, 3.3380e-11, 7.9661e-10,\n 8.1523e-11, 3.8542e-13, 9.4723e-11, 3.7131e-10, 2.1426e-12, 1.1881e-12,\n 5.9451e-13, 3.0219e-12, 3.6044e-11, 1.4214e-13, 3.5019e-11, 1.6183e-12,\n 3.0099e-10, 6.7689e-13, 2.6932e-11, 1.0338e-09, 6.1047e-12, 1.0299e-12,\n 8.1083e-11, 6.9074e-10, 4.0459e-10, 2.4708e-12, 2.3602e-13, 1.7440e-12,\n 1.3173e-11, 3.5149e-12, 9.0750e-12, 4.9704e-13, 7.0441e-12, 4.2788e-12,\n 1.7136e-12, 6.0018e-12, 2.7961e-13, 6.7091e-14, 7.2272e-11, 2.2909e-10,\n 8.1610e-10, 1.0074e-12, 1.0203e-12, 1.7468e-10, 5.6713e-10, 6.1799e-13,\n 6.7254e-11, 1.5303e-12, 5.2935e-13, 5.6877e-10, 5.4619e-11, 1.5779e-14,\n 1.1343e-10, 1.9116e-13, 6.1710e-13, 1.0299e-10, 5.6031e-14, 5.5382e-13,\n 9.8342e-14, 4.7551e-14, 7.8315e-14, 1.7237e-12, 2.5772e-10, 7.4068e-12,\n 2.1931e-13, 3.9871e-12, 1.2181e-11, 3.0434e-11, 1.4557e-10, 2.7277e-13,\n 6.7140e-10, 1.2995e-12, 1.6918e-11, 5.3827e-11, 4.5638e-11, 2.1063e-12,\n 2.3049e-10, 8.5014e-10, 4.1092e-13, 2.9236e-13, 6.4969e-13, 5.1055e-13,\n 8.8978e-12, 3.3713e-13, 2.6114e-13, 8.6287e-11, 8.9001e-11, 7.1124e-13,\n 6.2916e-13, 3.7079e-13, 1.5050e-10, 7.4136e-13, 1.5640e-10, 4.5740e-11,\n 1.7669e-12, 4.1883e-12, 1.0534e-10, 2.4489e-10, 1.9798e-12, 6.1392e-12,\n 2.3743e-13, 4.4600e-10, 1.8476e-11, 3.9345e-13, 3.2213e-12, 6.2675e-11,\n 7.2208e-13, 6.7008e-10, 5.8854e-13, 7.1583e-12, 2.6071e-12, 7.9638e-11,\n 2.6789e-11, 3.4876e-09, 3.9657e-10, 9.3500e-13, 1.7190e-09, 1.2922e-10,\n 3.2012e-11, 1.0739e-11, 8.2762e-14, 1.5392e-11, 5.3926e-12, 1.5008e-11,\n 2.4875e-12, 2.3124e-12, 9.2197e-13, 1.5065e-10], device='cuda:0')" + }, + "37": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.1073e-15, -9.4283e-14, 1.9183e-14, -7.3418e-16, -1.0636e-14,\n -1.2704e-14, -8.0541e-15, -3.1420e-15, -1.4629e-15, -5.3887e-15,\n 3.5311e-15, -1.5581e-15, 2.4386e-15, -1.5296e-15, -2.3082e-14,\n -4.7958e-15, -8.5347e-16, 2.4982e-16, 2.1976e-15, 3.7372e-15,\n -2.5068e-15, -2.7609e-14, 5.2201e-15, -4.1207e-15, -9.7467e-17,\n -2.1424e-14, -2.3648e-14, -3.5323e-15, 3.0219e-16, 3.4874e-16,\n 6.4341e-16, 1.6327e-14, 1.5184e-15, -7.0169e-14, -2.1779e-15,\n -1.4403e-15, -1.2474e-15, 1.1958e-15, -1.5770e-14, 3.0492e-15,\n 5.4600e-16, 1.4563e-14, -8.1041e-15, 1.2445e-16, -3.8367e-14,\n -5.6695e-16, 8.2002e-16, 5.8223e-15, 2.7198e-15, -3.4064e-14,\n 8.2762e-17, 1.2244e-14, 4.5757e-15, 2.5220e-15, -8.1414e-16,\n -6.9363e-15, -4.3767e-14, -4.8546e-16, -7.6140e-15, 8.3522e-16,\n -4.6745e-14, -1.5406e-15, -2.3849e-16, -5.5282e-16, -1.1835e-14,\n 2.2427e-16, -2.9465e-14, 3.5665e-14, -1.3969e-14, 9.5722e-15,\n -2.5948e-14, -4.9652e-16, 5.5689e-15, -3.1030e-15, -3.1481e-17,\n -2.0799e-15, -1.6049e-14, -2.1092e-16, -1.9539e-14, -1.2818e-14,\n 3.8532e-15, -1.0384e-14, 5.4161e-16, -6.7621e-15, -7.4373e-15,\n -1.4824e-14, -1.9243e-14, -3.2788e-14, -9.7164e-15, 5.3588e-17,\n 2.2168e-16, -2.2491e-15, -3.5978e-14, -4.1436e-15, -2.2029e-14,\n -4.1789e-14, -6.2273e-15, -4.7952e-15, -1.2829e-14, 1.0028e-14,\n -6.0268e-15, -7.0288e-15, -4.5025e-15, -2.0237e-14, -6.1644e-15,\n -1.1585e-15, 3.2201e-15, 4.4297e-14, 7.9007e-15, -1.3069e-14,\n -9.0363e-15, -2.3413e-15, -5.5859e-16, -7.9505e-15, 7.8029e-15,\n -1.2799e-15, -1.3389e-15, -2.7664e-16, 4.8750e-16, -1.4776e-14,\n -3.9896e-15, -1.2244e-15, -7.4864e-15, -1.1080e-14, 8.4792e-16,\n -5.7204e-16, 2.4914e-16, 1.0236e-16, -2.2887e-14, -1.1357e-14,\n 8.4274e-16, -2.0376e-14, 4.2463e-17, 2.8070e-15, 5.1909e-16,\n -3.8524e-14, -4.0380e-15, -7.2709e-15, -2.4763e-14, -4.6104e-15,\n -1.2369e-14, -1.8815e-14, -1.3258e-16, -2.4440e-16, -7.6157e-15,\n 5.9613e-16, -1.5426e-15, 3.9466e-15, -8.0534e-16, 4.8086e-15,\n 5.1303e-15, -4.2274e-15, -1.3257e-15, -4.0160e-16, 1.4814e-14,\n 5.0805e-17, -3.6582e-15, -2.8148e-14, -4.8238e-15, -1.4071e-17,\n 1.0573e-14, 3.0020e-16, -3.3540e-14, 6.1802e-16, -6.8053e-15,\n 2.2535e-15, -4.3958e-15, -6.6918e-16, -2.8909e-15, 1.7861e-14,\n -2.7006e-15, 7.0582e-15, -1.7339e-16, -1.3029e-14, -3.9378e-14,\n -4.9593e-16, 4.1766e-16, -1.8636e-15, 5.1727e-15, -8.1079e-15,\n 5.6652e-16, 1.7781e-14, -5.1041e-14, -6.6521e-15, -3.3213e-14,\n 5.4217e-15, -1.3714e-14, -2.5937e-15, 2.2343e-15, 1.1459e-14,\n 3.9169e-15, 1.7964e-15, -4.2597e-16, 8.1538e-15, 1.3515e-14,\n -1.9346e-15, -1.4594e-15, -1.3008e-15, 4.9728e-15, -1.4563e-14,\n -2.1340e-14, -6.9430e-16, -1.4006e-15, -4.3184e-14, -1.0908e-15,\n 2.9074e-14, -8.0801e-15, 1.4938e-15, 1.7038e-14, 8.3951e-15,\n 3.0598e-15, 1.6506e-15, -9.9853e-15, -1.6163e-14, 9.9481e-15,\n -7.9742e-16, -7.5077e-16, 3.4688e-15, -2.0044e-15, 1.6838e-15,\n -9.4036e-15, 5.0919e-15, -2.2775e-14, -8.5284e-15, -5.3017e-15,\n -1.2327e-14, 6.3353e-15, -5.7496e-14, 2.9145e-15, -4.3861e-15,\n -4.0728e-14, 1.0926e-15, 8.9747e-15, -3.2995e-14, -5.6195e-16,\n 2.7402e-14, -2.8494e-15, 2.0566e-14, -3.0943e-15, 5.5217e-15,\n -8.2663e-15, 1.6707e-14, -8.5979e-16, -2.4757e-14, 1.2813e-14,\n 7.1649e-16, -7.9819e-15, 2.1761e-14, -5.0113e-14, -2.7134e-14,\n -2.6088e-15, 4.9478e-16, -5.0192e-15, -2.4207e-14, -1.6497e-14,\n 3.6358e-15, -2.1490e-14, 3.0968e-15, -1.0061e-14, -2.2540e-14,\n -1.2038e-14, 2.3189e-15, 2.4775e-15, 1.2906e-15, 4.8213e-15,\n -1.3239e-14, 2.5547e-14, -9.7949e-15, 1.2533e-14, 2.4527e-15,\n 3.0056e-15, -1.3368e-14, -6.5963e-15, -2.1609e-15, 4.7235e-16,\n -2.3889e-14, -3.8246e-14, -1.0556e-15, 8.8718e-15, -7.7746e-15,\n -9.2933e-15, 2.3466e-14, 4.3457e-15, -4.3168e-14, 2.9779e-15,\n -1.7806e-14, -5.2733e-16, -1.2060e-14, -2.7500e-14, 1.3420e-14,\n -2.6259e-14, 3.2944e-15, -4.2838e-14, 5.9793e-15, 4.3470e-16,\n -3.1146e-14, -1.6867e-14, -7.7590e-15, -4.9726e-15, -1.0048e-14,\n 3.0673e-15, -2.7692e-14, 1.3450e-15, 1.1090e-14, -3.0485e-14,\n 8.5932e-16, -8.2699e-15, 3.6299e-16, -2.7526e-14, 7.1623e-15,\n -2.5895e-14, 1.0508e-14, -4.8764e-14, 4.3704e-14, -1.2882e-15,\n -1.6442e-15, -3.0805e-14, 6.9803e-16, -3.3558e-14, 8.6448e-15,\n 2.0950e-15, -9.2971e-16, -3.7703e-17, 4.4142e-15, -1.0737e-14,\n -7.2795e-14, -1.2130e-15, -1.3519e-15, -3.6929e-16, -3.3009e-14,\n -3.0721e-15, -2.9505e-15, -8.8099e-15, -7.6802e-15, -1.1049e-17,\n 6.9673e-16, 9.5407e-15, -2.3041e-15, -3.7355e-14, -1.4801e-14,\n -9.1169e-15, 3.6658e-15, -1.5460e-14, 1.4997e-15, -2.4519e-14,\n -1.4076e-14, -6.8910e-16, 3.7425e-14, 5.8668e-16, -1.6671e-14,\n 1.1238e-15, -3.6871e-15, 5.8951e-16, -2.1359e-15, -7.5139e-16,\n -8.9242e-15, -3.0245e-14, 1.4183e-14, 1.2438e-15, -1.5445e-14,\n -6.8771e-15, -3.4008e-14, -9.1158e-15, -1.0708e-14, 1.0453e-16,\n 2.6318e-15, 5.0146e-15, 1.4389e-14, -7.6870e-15, -6.0324e-15,\n 2.0184e-15, 1.1938e-16, -3.0247e-15, 6.2836e-15, -2.3918e-14,\n -1.1666e-15, -3.6804e-15, -7.5119e-16, -3.6117e-15, 1.1545e-14,\n -4.7443e-15, -2.9673e-15, 1.5957e-16, -6.1308e-15, -1.5597e-14,\n 8.4441e-15, -1.8975e-14, 7.3334e-15, -1.1938e-14, 9.5502e-16,\n 7.9696e-15, 5.5906e-16, 1.4685e-14, 3.8493e-14, -5.1596e-15,\n -2.2428e-15, -2.3131e-14, -2.4921e-14, 7.3167e-15, -1.7173e-14,\n -6.2271e-15, 2.1306e-15, -1.2694e-15, 2.3268e-15, -1.8574e-14,\n -5.9714e-15, -1.8265e-15, -1.1546e-14, -1.6643e-14, -7.6325e-15,\n -3.8691e-14, -1.9864e-14, -2.2861e-14, 1.3552e-15, -6.4256e-15,\n -3.7888e-14, -6.8082e-15, 9.9966e-15, -2.7765e-14, 2.2076e-15,\n 1.9312e-14, -1.4803e-14, -2.5453e-14, -3.2025e-14, 2.6819e-15,\n -1.6583e-15, -6.1197e-15, -4.5208e-16, -7.1678e-15, -1.1053e-15,\n 1.0277e-15, -1.5364e-15, -1.5591e-14, 2.5744e-16, -1.0691e-15,\n -3.4023e-14, 6.0017e-15, -3.0139e-14, 1.8973e-15, -6.5364e-15,\n 5.8555e-16, -7.3482e-16, -1.8955e-15, -3.9369e-16, -2.7773e-15,\n -2.3960e-15, -1.1913e-14, 4.3544e-16, 4.8108e-15, -5.9823e-18,\n 4.3237e-16, 9.7273e-16, -2.7748e-14, -2.5742e-14, -3.0215e-15,\n -5.3133e-16, -2.3480e-15, -2.2589e-15, -2.1726e-15, -6.3988e-15,\n 3.4455e-16, 2.6111e-15, -4.8267e-15, -3.2065e-14, -4.1473e-14,\n 3.1098e-14, -1.5209e-15, -8.5828e-14, -7.1441e-15, 1.4773e-14,\n 2.5925e-15, 2.1776e-14, -1.3873e-14, 3.0893e-17, 2.3763e-14,\n 3.4732e-15, 3.0079e-14, 4.6263e-15, 3.4684e-16, -1.2360e-14,\n -4.1344e-15, 2.1030e-14, -1.5553e-15, -6.3087e-15, 1.1473e-15,\n -4.3722e-16, -1.2688e-15, -6.5604e-15, -1.0879e-14, -4.1853e-14,\n -2.3154e-15, -8.8561e-15, 5.8216e-16, -8.1969e-15, -1.6941e-14,\n -6.7615e-16, 1.1253e-15, -4.2711e-15, -1.4928e-14, 5.0577e-16,\n -2.7090e-15, -1.1192e-15, 1.5390e-14, -2.6190e-15, 1.6019e-15,\n -1.9654e-14, -1.5315e-14, -1.3118e-14, -6.0197e-15, -6.6704e-17,\n -1.5463e-15, -8.1736e-16, -1.6788e-16, -6.2790e-16, 1.9018e-16,\n -1.5516e-15, -1.4553e-14, -3.3324e-14, -9.8068e-15, 9.3319e-15,\n -1.5418e-14, 1.2498e-15, -3.3989e-16, -7.8360e-17, -2.3566e-14,\n -6.8553e-16, -3.5512e-15, 2.1687e-16, -2.7601e-14, -2.1510e-14,\n -9.5441e-15, 6.0478e-15, -1.3402e-14, 1.3589e-15, -7.6981e-15,\n -8.5180e-15, 2.1389e-14, 1.0652e-14, -2.9436e-16, -3.4058e-14,\n -4.6296e-14, 1.4644e-16, -2.0000e-15, 6.6259e-16, -2.3768e-14,\n -9.2802e-14, -5.4899e-16, -2.8487e-15, -1.2719e-14, 1.5587e-14,\n -1.0183e-14, 1.2786e-14, -6.9449e-15, -1.4194e-14, 7.7840e-17,\n 4.1087e-15, -6.0729e-15, -3.3335e-14, 1.3891e-15, -1.4359e-14,\n 2.4396e-15, -5.0967e-15, -1.2011e-15, -5.6935e-15, 5.5762e-16,\n -7.8209e-15, -7.1191e-17, -4.9238e-14, 7.8771e-15, -4.0242e-15,\n -7.2853e-16, 7.9400e-16, -4.0645e-15, 1.9049e-15, -7.3403e-14,\n -9.5846e-17, 1.9055e-15, 2.8642e-15, -1.9840e-14, 1.0925e-15,\n -5.0374e-15, -2.1684e-16, -2.9876e-14, -1.5374e-14, -2.1689e-14,\n -2.2329e-14, -2.1841e-14, -9.5844e-15, -7.2645e-16, 1.0028e-14,\n 8.2579e-16, 1.8958e-15, -2.2954e-14, -8.6203e-16, 3.4172e-16,\n 4.9368e-17, -1.3785e-15, -3.4118e-15, -1.0670e-14, -5.2345e-16,\n -1.7497e-14, -8.8890e-15, -1.6601e-14, 1.6564e-15, 2.7254e-15,\n 4.4927e-15, -4.5451e-14, 4.0697e-15, -8.6628e-15, -4.6941e-14,\n 6.9389e-15, -1.1504e-15, 7.0242e-16, -3.2745e-14, -7.7820e-15,\n -4.4596e-14, -3.5063e-14, -1.0894e-14, 7.2486e-16, 1.2783e-14,\n 8.6259e-15, 6.1200e-15, -3.3471e-14, -3.9315e-14, 3.5094e-14,\n 5.9438e-15, -1.2624e-14, -2.2525e-14, -2.8507e-15, -2.8794e-14,\n 5.0808e-15, 1.8626e-15, 5.5594e-15, -2.6338e-15, -1.6344e-15,\n -1.7896e-14, -6.4897e-17, 6.8580e-17, -1.0680e-14, -1.8238e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.2223e-10, 7.1773e-12, 1.6047e-11, 1.8420e-12, 6.7424e-12, 2.5249e-10,\n 5.0243e-10, 4.0917e-11, 5.5536e-13, 1.2137e-11, 2.5338e-11, 3.1535e-12,\n 1.1376e-10, 1.3618e-12, 5.7383e-11, 3.9997e-12, 2.4498e-10, 2.2075e-12,\n 1.9149e-10, 5.5535e-13, 1.2441e-12, 2.1663e-11, 8.5895e-11, 1.6513e-12,\n 1.0696e-13, 6.4409e-11, 5.7395e-10, 3.2604e-10, 1.4773e-11, 2.2735e-12,\n 1.5516e-10, 1.9305e-11, 4.7277e-12, 8.0696e-12, 2.3326e-12, 9.3013e-11,\n 6.6973e-12, 2.0156e-13, 8.6817e-11, 1.2247e-12, 6.2950e-12, 2.3813e-10,\n 3.4878e-11, 1.2868e-13, 5.4300e-12, 7.2663e-12, 1.3602e-12, 1.1615e-10,\n 2.8279e-11, 1.2841e-10, 6.3994e-13, 1.7905e-10, 2.4992e-12, 6.2827e-11,\n 1.4512e-10, 6.2700e-10, 6.6156e-10, 4.0359e-12, 4.3471e-10, 1.0021e-12,\n 9.5396e-11, 1.2774e-11, 2.8201e-11, 1.6358e-11, 6.0504e-11, 3.0234e-12,\n 3.7526e-10, 2.7474e-10, 5.5214e-11, 3.4875e-10, 9.0128e-10, 5.1516e-12,\n 7.8875e-12, 8.3420e-13, 6.4494e-11, 1.0422e-11, 2.5432e-11, 2.0281e-12,\n 5.4928e-11, 3.5273e-10, 1.9279e-12, 7.5335e-12, 7.2010e-12, 1.0615e-10,\n 3.6745e-12, 4.5975e-12, 1.6404e-11, 4.4304e-11, 3.3849e-10, 7.5633e-12,\n 1.0227e-11, 1.6355e-11, 4.9912e-11, 1.9688e-10, 3.0080e-10, 5.6381e-10,\n 2.5857e-11, 3.0053e-10, 4.4570e-11, 3.7869e-11, 1.8395e-12, 1.8850e-10,\n 8.7852e-12, 1.0787e-10, 2.4433e-10, 1.2680e-12, 2.3480e-11, 2.5846e-10,\n 9.0005e-13, 1.3070e-11, 2.2743e-10, 2.0517e-11, 4.2668e-12, 8.0020e-13,\n 1.8673e-13, 2.8260e-12, 8.4938e-13, 1.3785e-11, 2.1820e-13, 1.9211e-12,\n 3.7266e-10, 1.3501e-10, 5.3648e-11, 1.8239e-10, 1.5073e-12, 9.4106e-11,\n 1.5357e-13, 1.3114e-12, 7.9577e-13, 2.6131e-11, 1.1692e-12, 1.1222e-11,\n 2.7879e-10, 1.5727e-10, 9.6079e-12, 7.1180e-10, 5.0260e-11, 1.1076e-10,\n 2.4472e-10, 3.4716e-10, 1.4685e-10, 3.8012e-11, 9.9585e-13, 1.1455e-12,\n 2.1932e-12, 6.8925e-12, 2.1194e-11, 5.4951e-10, 1.8694e-10, 1.6388e-12,\n 6.3904e-11, 1.4402e-11, 6.7220e-12, 1.7924e-13, 9.2213e-11, 2.8933e-11,\n 2.5337e-11, 1.3845e-10, 4.9538e-10, 5.0391e-11, 5.4928e-10, 6.0045e-13,\n 2.4251e-10, 4.1150e-13, 9.2719e-11, 2.0950e-12, 3.8344e-13, 6.4437e-12,\n 1.9606e-11, 2.8722e-10, 2.1507e-13, 3.6554e-11, 1.1032e-11, 1.9343e-10,\n 1.6466e-10, 6.1455e-11, 5.5775e-11, 1.1521e-09, 2.0758e-11, 5.5320e-10,\n 4.6802e-13, 1.7848e-11, 1.6446e-10, 1.0969e-11, 1.0962e-10, 2.8073e-13,\n 5.0033e-10, 4.6504e-10, 2.4866e-12, 2.6525e-12, 1.3534e-12, 2.9058e-12,\n 6.6736e-12, 2.2296e-13, 4.5307e-10, 1.6467e-12, 3.8239e-12, 2.4182e-13,\n 7.2361e-10, 3.5826e-10, 2.3952e-10, 1.5059e-11, 6.3978e-12, 9.7107e-12,\n 3.5517e-11, 3.7335e-10, 5.1165e-11, 4.9949e-13, 8.9018e-10, 5.4694e-10,\n 2.4732e-11, 1.3775e-11, 2.6146e-12, 8.2455e-11, 5.6309e-10, 5.7984e-13,\n 3.2628e-12, 1.9306e-12, 9.8458e-11, 1.0286e-11, 3.0480e-11, 8.6744e-13,\n 7.0406e-10, 5.3882e-12, 1.8155e-10, 2.0954e-10, 3.0913e-11, 1.5132e-10,\n 1.9569e-12, 2.0299e-10, 4.1823e-12, 1.0676e-12, 1.1563e-10, 3.8443e-11,\n 4.4375e-10, 6.3697e-10, 3.5570e-11, 5.4892e-11, 1.4235e-12, 2.3194e-11,\n 1.4475e-10, 1.9612e-12, 1.6817e-11, 1.1302e-11, 1.4089e-12, 6.8041e-13,\n 8.9807e-11, 4.0826e-10, 5.7801e-10, 5.9573e-10, 1.2729e-10, 1.0258e-11,\n 3.1545e-12, 3.9755e-12, 7.8713e-11, 2.4868e-10, 1.8801e-10, 2.6053e-12,\n 2.5801e-11, 1.7976e-10, 6.7402e-11, 6.3554e-13, 6.6480e-11, 3.3803e-11,\n 3.2272e-10, 2.6536e-11, 5.0410e-12, 1.2611e-10, 4.8195e-10, 2.5449e-12,\n 5.4952e-11, 1.4667e-12, 1.0189e-10, 8.0638e-11, 3.5900e-12, 1.1036e-12,\n 5.2256e-11, 2.0157e-13, 1.4494e-10, 4.3545e-10, 2.2198e-10, 1.6407e-11,\n 2.0309e-11, 2.2556e-11, 5.2122e-11, 1.6805e-11, 5.2284e-13, 2.1014e-11,\n 7.8829e-11, 6.1176e-13, 1.4032e-11, 2.4142e-12, 9.8056e-12, 1.3523e-12,\n 5.8034e-13, 3.6340e-10, 2.3831e-10, 9.4839e-12, 2.0425e-11, 7.7175e-13,\n 2.9887e-10, 2.0203e-11, 7.0691e-12, 2.4534e-10, 1.4074e-10, 6.6167e-12,\n 2.3751e-11, 5.3576e-12, 1.7512e-10, 4.3635e-11, 6.4725e-11, 3.5579e-10,\n 8.5323e-10, 5.5908e-10, 1.0785e-10, 8.8537e-12, 2.1470e-11, 2.9985e-11,\n 5.8732e-10, 1.2030e-09, 1.6829e-12, 3.5254e-12, 4.1027e-13, 1.5259e-10,\n 2.9427e-10, 2.9879e-11, 5.4636e-11, 6.5456e-14, 6.9081e-13, 4.7935e-10,\n 8.3016e-11, 2.4536e-11, 2.6769e-11, 4.3183e-12, 4.8281e-10, 2.3377e-11,\n 1.2610e-09, 1.3911e-13, 4.8737e-10, 8.8015e-10, 2.2851e-10, 5.8660e-13,\n 4.2286e-11, 9.4916e-13, 1.0239e-10, 1.7205e-10, 4.8663e-11, 5.9484e-10,\n 9.4000e-13, 3.3318e-10, 1.7140e-13, 4.0861e-11, 1.6890e-12, 3.1213e-11,\n 3.9298e-13, 1.9912e-10, 2.2294e-11, 1.0432e-10, 4.4187e-13, 2.4866e-11,\n 6.7972e-12, 1.6856e-10, 7.6281e-12, 2.3269e-12, 5.2115e-12, 3.3516e-11,\n 1.0091e-11, 5.4560e-11, 1.5336e-10, 7.1945e-11, 3.6861e-10, 2.8229e-11,\n 2.8746e-12, 8.3421e-11, 1.6438e-12, 1.8651e-11, 1.4535e-10, 9.1871e-11,\n 1.3424e-11, 2.7054e-10, 1.8023e-11, 1.1122e-12, 3.5709e-12, 1.3158e-12,\n 1.8337e-12, 1.5375e-11, 3.3510e-11, 1.9792e-11, 3.2948e-11, 3.1873e-13,\n 9.2863e-11, 3.7214e-12, 7.6719e-10, 6.7669e-10, 3.3406e-12, 1.4553e-10,\n 3.5779e-11, 2.5567e-11, 1.2707e-11, 3.6021e-10, 2.5253e-13, 1.5692e-11,\n 7.6857e-12, 1.4516e-10, 9.8605e-12, 5.2331e-13, 1.9362e-11, 1.3984e-11,\n 6.2064e-11, 6.4425e-11, 2.1484e-11, 2.8418e-11, 5.6112e-13, 4.9961e-11,\n 1.9806e-11, 2.1278e-10, 4.7889e-10, 2.4424e-11, 1.1795e-10, 7.7592e-13,\n 6.4600e-12, 4.0003e-10, 9.0582e-12, 2.2215e-12, 3.3987e-12, 1.8884e-12,\n 2.2395e-13, 5.4917e-11, 2.8035e-10, 4.1043e-12, 2.7066e-13, 7.9831e-10,\n 2.9778e-10, 9.6545e-13, 5.1430e-11, 5.7446e-11, 2.4581e-11, 3.9028e-11,\n 1.7679e-11, 9.9110e-12, 1.7347e-10, 6.3039e-12, 6.6662e-12, 3.2916e-12,\n 9.1319e-11, 5.1351e-10, 5.4487e-10, 2.9397e-12, 1.4432e-10, 8.9225e-14,\n 3.3309e-12, 3.5071e-12, 3.0493e-11, 1.3020e-12, 3.9530e-12, 8.6521e-11,\n 1.2240e-10, 7.2112e-12, 1.0035e-12, 6.1854e-11, 4.0744e-11, 6.9277e-13,\n 2.4707e-12, 5.6753e-11, 7.1608e-10, 3.9686e-10, 5.0627e-12, 9.4780e-12,\n 1.0235e-12, 2.0776e-11, 1.8581e-10, 3.2724e-10, 5.2002e-12, 3.4808e-12,\n 9.8422e-12, 4.0329e-12, 1.7390e-12, 6.9928e-11, 1.8614e-11, 3.3343e-11,\n 1.5293e-11, 5.9527e-11, 2.5599e-12, 2.6049e-10, 1.0707e-12, 5.6685e-10,\n 6.4451e-10, 3.0959e-11, 4.7281e-11, 2.2970e-11, 2.0617e-12, 5.3854e-11,\n 3.3170e-12, 1.5950e-11, 2.1143e-10, 5.6947e-13, 3.7873e-12, 3.9324e-10,\n 2.3265e-10, 4.5671e-13, 2.9134e-10, 2.8080e-11, 2.3251e-11, 1.6531e-11,\n 2.4487e-11, 1.0146e-10, 1.4313e-10, 6.1753e-11, 1.3480e-10, 5.2273e-13,\n 3.0554e-12, 2.1722e-10, 6.1151e-14, 9.4136e-13, 1.7989e-11, 3.6912e-10,\n 6.5881e-11, 3.1283e-12, 5.3356e-10, 9.8183e-11, 5.4957e-11, 1.7966e-11,\n 1.0109e-11, 1.7582e-12, 1.5240e-11, 2.3283e-12, 1.0419e-10, 2.2269e-11,\n 9.5259e-11, 2.3863e-12, 1.7903e-10, 1.6406e-09, 5.7875e-11, 9.5917e-12,\n 1.1087e-10, 4.6587e-10, 5.6477e-10, 1.6967e-10, 1.3959e-13, 5.5354e-12,\n 1.5079e-11, 6.8735e-13, 8.7673e-11, 1.3862e-12, 6.1114e-12, 6.8492e-12,\n 2.2726e-11, 3.0438e-12, 7.7161e-12, 6.8690e-13, 5.2800e-10, 2.3327e-10,\n 3.4253e-10, 1.4477e-11, 1.0515e-11, 7.2293e-10, 1.0930e-10, 1.0383e-12,\n 1.3963e-10, 1.2889e-11, 2.7232e-12, 5.4557e-10, 6.6143e-11, 4.6965e-13,\n 6.5486e-10, 2.5327e-12, 7.4828e-13, 4.8564e-10, 1.0069e-12, 2.6561e-12,\n 5.6445e-13, 5.8170e-13, 4.5941e-13, 5.3988e-12, 8.6853e-10, 1.6228e-12,\n 1.0789e-12, 1.9302e-11, 2.2961e-11, 9.2139e-12, 1.3086e-10, 1.7329e-13,\n 5.2870e-10, 3.4110e-12, 1.1123e-11, 8.3796e-11, 5.0603e-11, 1.1110e-11,\n 1.5679e-10, 6.0622e-10, 1.5915e-11, 2.7986e-12, 2.1856e-11, 2.6802e-13,\n 1.1488e-11, 2.0492e-13, 6.3856e-13, 5.9629e-11, 9.7296e-11, 2.5049e-13,\n 2.2829e-11, 5.4895e-12, 2.4541e-10, 7.1324e-12, 5.9250e-10, 1.5352e-11,\n 2.3091e-11, 4.4098e-12, 1.0062e-10, 5.8455e-10, 1.4619e-11, 2.7759e-11,\n 3.4435e-12, 1.2642e-10, 1.6994e-10, 8.8501e-12, 1.0773e-10, 3.7306e-11,\n 4.0377e-13, 6.7857e-10, 4.0276e-12, 2.5017e-12, 1.1568e-11, 1.9448e-11,\n 2.5634e-10, 1.6163e-09, 6.8126e-10, 3.6224e-12, 9.3824e-10, 5.4453e-10,\n 1.6742e-10, 4.3081e-12, 1.1930e-12, 4.0845e-11, 3.4897e-12, 2.4105e-11,\n 6.6686e-13, 6.0690e-11, 5.3999e-12, 6.2914e-10], device='cuda:0')" + }, + "38": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-1.4952e-17, -3.7730e-18, 5.6052e-45, ..., -1.6350e-17,\n -1.7467e-15, -5.6052e-45],\n [-1.0526e-16, -1.5680e-17, -5.6052e-45, ..., -3.2391e-15,\n -1.0464e-14, -5.6052e-45],\n [ 2.6772e-17, 1.3555e-17, 5.6052e-45, ..., 1.2921e-15,\n 3.3784e-15, 5.6052e-45],\n ...,\n [-9.3315e-18, 3.2967e-18, 5.6052e-45, ..., 2.6801e-16,\n -1.0402e-16, -5.6052e-45],\n [-1.4759e-17, -9.3557e-19, -5.6052e-45, ..., -9.6631e-16,\n -2.6586e-15, -5.6052e-45],\n [ 2.5686e-18, 3.3699e-18, -5.6052e-45, ..., 2.3864e-16,\n 1.3405e-16, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.0766e-12, 1.7637e-12, 9.2226e-17, ..., 2.9174e-12, 6.9746e-13,\n 3.1346e-14],\n [2.8036e-12, 1.8390e-12, 1.4587e-16, ..., 4.1913e-12, 9.6052e-13,\n 1.6745e-13],\n [1.6639e-12, 7.4385e-13, 3.6628e-15, ..., 1.4876e-12, 6.2716e-13,\n 6.9678e-14],\n ...,\n [1.0999e-13, 6.7498e-14, 1.3862e-16, ..., 1.6275e-13, 1.3043e-13,\n 6.3670e-15],\n [1.9891e-13, 1.1973e-13, 5.0520e-17, ..., 1.9872e-13, 3.2734e-14,\n 6.2674e-15],\n [4.3571e-13, 3.6010e-13, 3.6139e-18, ..., 8.0421e-13, 1.9061e-13,\n 9.8403e-15]], device='cuda:0')" + }, + "39": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.3505e-13, -1.1417e-12, 7.8217e-13, 5.6415e-13, -1.6195e-13,\n -3.8182e-13, 8.3726e-14, -9.5214e-14, 2.7175e-13, -7.3581e-14,\n 3.4064e-14, 5.0764e-14, 8.7128e-14, -1.8689e-13, -1.3871e-13,\n -2.7120e-13, -1.9337e-13, 8.2970e-15, 3.8125e-13, 1.5009e-13,\n 2.0786e-13, 6.6493e-14, 1.3173e-13, 3.0110e-13, -7.2451e-15,\n -7.9062e-14, 1.4348e-13, 1.4873e-13, 3.0957e-14, 6.9047e-14,\n 8.4106e-13, 2.2600e-13, -2.5391e-14, -4.1910e-13, 1.0282e-13,\n -4.3222e-13, -3.4756e-13, 3.4976e-13, -5.4310e-13, 1.2912e-13,\n 3.9643e-14, 2.1912e-14, 7.0314e-14, 5.6381e-14, -6.3425e-13,\n 3.8967e-14, -3.2156e-14, -1.8896e-14, -2.6693e-14, 5.1137e-13,\n 1.1224e-13, 1.7617e-13, 4.2744e-13, 5.6536e-13, 8.5436e-14,\n -5.1323e-14, -2.3812e-13, 4.9960e-14, -1.9319e-13, -6.3835e-15,\n -3.1363e-13, -2.9163e-13, -6.2613e-14, 8.0254e-14, 5.9610e-14,\n -1.2758e-13, -6.1647e-14, 2.8496e-13, 1.1642e-14, 4.3891e-13,\n 1.3390e-13, 8.0841e-13, 8.6973e-14, 2.1689e-13, 1.7277e-13,\n 1.0355e-13, -3.1019e-14, 1.4852e-13, -7.8060e-13, 3.8630e-13,\n 2.7631e-13, -1.7307e-13, 3.3898e-13, 9.7604e-14, 4.2957e-14,\n -1.1655e-13, -8.1974e-14, -4.9047e-13, -1.0065e-14, 1.6783e-13,\n 7.2694e-14, 1.1027e-13, -9.0476e-15, 5.2264e-13, -7.4792e-13,\n 2.1261e-13, -5.9005e-13, -4.6938e-13, 3.4982e-13, 2.5816e-13,\n 1.0113e-13, 2.8524e-13, -3.6026e-14, 2.0329e-13, -2.2116e-14,\n -4.7535e-14, 1.0759e-13, 1.1949e-12, 6.7486e-14, 4.1092e-15,\n 1.2006e-14, -1.7963e-13, -1.2421e-13, 4.9764e-14, 2.5817e-13,\n 1.6379e-14, 3.3424e-13, 2.6089e-13, -1.7797e-13, 2.1396e-13,\n 2.4355e-13, 2.6493e-13, 1.6566e-13, 4.1347e-14, 4.7142e-14,\n -2.1833e-13, -1.7130e-13, 1.7697e-13, -6.0883e-15, -4.2612e-13,\n -3.4800e-14, 9.6271e-15, 9.0637e-14, -4.2825e-13, -1.1647e-13,\n -2.6596e-13, -1.1564e-13, 6.6692e-13, 4.6657e-14, 4.9534e-14,\n -1.4057e-13, -1.3004e-12, 5.6556e-14, -1.0584e-14, 1.0864e-13,\n 1.8115e-13, 2.7024e-13, -2.6201e-13, 2.8579e-13, 1.0754e-13,\n 1.6908e-13, 2.9003e-13, -5.0715e-14, -1.1854e-13, 4.7583e-14,\n 2.4885e-14, 2.3489e-13, 1.8877e-14, 1.0945e-13, 3.6389e-13,\n 7.6167e-14, 2.2110e-13, -3.1958e-14, 2.0856e-15, 5.0213e-14,\n -1.8049e-13, 1.5651e-13, -3.4049e-13, 4.3150e-13, 2.0874e-13,\n 6.2685e-13, 1.7866e-14, 5.2523e-14, 1.5515e-14, -5.8343e-14,\n 2.8083e-14, 7.6022e-14, -3.8490e-13, -1.8282e-14, -1.0457e-13,\n -1.9735e-13, 2.9071e-13, -4.0225e-13, 1.6159e-13, -8.4238e-14,\n 5.2312e-14, -3.5923e-13, 1.9973e-13, -2.3276e-13, 1.8577e-14,\n 3.9935e-14, -2.1572e-13, -5.3160e-13, 1.7534e-13, 4.5272e-13,\n -1.3884e-13, 2.8351e-13, -7.1388e-15, 8.8349e-14, 2.7094e-14,\n 5.2363e-13, 6.3359e-14, 1.6666e-13, -3.1801e-13, -4.2147e-13,\n 3.7059e-13, 5.7462e-14, -4.2108e-14, 1.9834e-13, -2.5059e-13,\n -1.7495e-13, 8.1731e-14, -4.6210e-13, 1.5518e-13, 1.1886e-13,\n 2.3177e-13, -2.2022e-13, -4.0430e-13, 1.2443e-13, -1.4718e-13,\n -1.6396e-13, -5.1029e-15, -2.2988e-13, 1.3542e-13, -4.6959e-14,\n -1.4335e-13, 4.4829e-13, -3.1465e-13, -2.5937e-13, 2.1976e-14,\n -4.6072e-13, -3.1155e-16, 1.2706e-13, -3.0671e-13, 8.3612e-14,\n 2.3437e-13, 1.1227e-13, 1.5901e-13, -4.5620e-13, 1.6400e-13,\n 3.2921e-14, -2.7633e-13, -3.3366e-13, -1.5788e-13, 5.8118e-13,\n 1.6267e-13, 4.3002e-13, 2.5598e-13, -1.2210e-13, 1.2839e-13,\n -9.1458e-14, 1.6561e-14, 1.5868e-13, -2.2047e-13, 3.0148e-13,\n -5.2034e-14, -1.0864e-13, 4.6253e-13, -1.1617e-13, 2.2104e-13,\n 1.4954e-14, -1.0164e-13, 1.8818e-13, -2.7169e-13, 1.5541e-13,\n 7.5762e-14, 1.9348e-13, -6.6852e-13, 4.1028e-14, -5.6511e-13,\n 1.2221e-13, -6.6280e-14, 1.8665e-13, -2.6218e-13, -1.6825e-13,\n -7.3386e-14, -4.1585e-13, -6.8253e-14, 4.6215e-13, -3.5069e-13,\n 3.4818e-13, 6.9437e-13, 1.6574e-13, -3.4100e-13, 1.6095e-13,\n 2.2602e-13, 5.1533e-14, -9.9801e-14, 3.3070e-13, 2.1149e-14,\n 2.7424e-13, -2.8868e-13, -8.4037e-13, 7.0720e-13, 7.8628e-14,\n -1.7960e-14, -3.5018e-14, -1.1430e-13, 6.4609e-14, -1.9431e-14,\n 3.6009e-13, 4.1879e-14, -2.6118e-14, 5.1976e-14, -6.5445e-13,\n -2.5658e-14, 4.0352e-14, -9.6119e-14, -4.3445e-14, -5.1571e-13,\n 3.6944e-14, 1.1613e-13, -4.6836e-13, 4.5494e-13, 1.1012e-13,\n -4.2740e-13, -1.5152e-13, 6.3092e-13, -2.8193e-13, 2.3566e-13,\n -2.3815e-14, 5.2951e-13, 3.4526e-14, 5.7553e-14, 2.2258e-13,\n 9.8541e-14, 1.8429e-13, 3.1456e-14, 6.9673e-14, 1.3576e-13,\n 3.3213e-14, 9.2539e-14, 2.5121e-13, 1.5706e-14, -7.5712e-14,\n 2.8692e-13, -1.0714e-13, -1.3995e-14, 7.1065e-14, -5.6037e-13,\n 1.6457e-13, 5.5301e-13, -3.7571e-13, 9.4303e-14, -1.3383e-12,\n -3.8247e-13, 4.7796e-14, 1.5842e-13, -5.1158e-13, 2.3370e-14,\n 5.3222e-13, 1.0711e-13, 3.0497e-13, -4.8753e-14, 2.2217e-13,\n -9.4530e-14, -4.3770e-13, -9.8110e-14, -2.4392e-13, 1.1941e-13,\n 1.4690e-13, -1.9795e-13, 1.1776e-13, -1.0641e-12, -9.4748e-14,\n 1.6378e-13, 1.2580e-13, -1.2855e-13, 2.3457e-13, -7.1835e-13,\n 1.0913e-13, 6.4375e-14, -1.5839e-13, 8.3916e-14, 1.5877e-13,\n 1.1317e-13, 1.3787e-13, 1.9499e-13, -5.9528e-15, 1.6996e-13,\n -1.1220e-13, 1.1987e-13, -3.8109e-14, 2.9554e-13, -1.8688e-14,\n 4.1413e-14, -3.8327e-13, 1.9287e-13, -5.9668e-15, -6.2545e-14,\n -4.2707e-14, 1.6322e-13, -4.0153e-13, -6.3108e-15, -8.9317e-13,\n 1.4485e-13, -7.7645e-13, -1.2147e-12, 1.3149e-13, -1.5045e-13,\n -3.0345e-13, 2.4560e-14, -1.3415e-13, -7.4686e-14, -3.4109e-13,\n -2.9266e-13, 4.4045e-13, -2.3959e-13, 3.8592e-13, 3.3619e-14,\n 5.0724e-13, 1.5940e-13, -1.1516e-12, 1.9852e-13, 1.0166e-13,\n -1.4873e-13, 1.4259e-13, -7.8673e-14, -3.1945e-13, -2.1552e-13,\n -1.6412e-13, 5.3752e-15, -2.8188e-13, -1.9473e-13, -4.3430e-15,\n 6.5736e-13, -1.1194e-13, 1.7014e-13, 7.4248e-15, 3.4303e-13,\n -2.0115e-13, 1.4131e-13, 2.7775e-14, 2.8420e-13, 8.4193e-14,\n 6.4149e-14, 8.8334e-14, -1.9261e-13, 1.5968e-13, 1.8175e-13,\n 1.8160e-13, 1.7976e-14, -1.6159e-13, 2.1752e-13, 7.0641e-14,\n -1.8404e-13, -7.1890e-13, 3.1655e-14, 1.4303e-13, -2.3579e-13,\n -1.8343e-13, 6.6705e-14, -1.3127e-14, 1.1263e-13, 1.0626e-12,\n -1.6104e-13, -4.5125e-14, -2.7787e-13, 2.1607e-13, -1.1786e-13,\n 4.2117e-13, 1.4277e-13, -6.5673e-13, 2.5929e-13, 7.0583e-15,\n 1.9287e-13, 1.7625e-13, -6.1175e-13, -2.6478e-13, 2.5600e-13,\n -1.0385e-14, 3.2673e-13, -1.1407e-13, 1.1926e-13, 4.0988e-14,\n 7.1468e-14, 3.2473e-13, 7.4558e-14, 6.1587e-14, 4.5705e-14,\n -1.0282e-13, -1.4717e-13, -9.7882e-14, -1.8691e-13, 4.2030e-14,\n -2.8097e-13, 2.6207e-13, 9.9377e-14, -1.0029e-13, -2.1000e-13,\n 1.4203e-13, 5.9517e-14, 3.4519e-14, 5.4017e-14, -1.3042e-13,\n -2.4669e-13, -1.6073e-13, 1.7144e-13, -2.5461e-14, 2.7007e-13,\n -1.0513e-13, 6.6553e-14, 1.7202e-13, 1.9896e-14, 3.8060e-14,\n -4.5535e-14, 3.0718e-13, -6.9235e-14, 8.4165e-13, -1.9475e-13,\n -9.3413e-14, 1.3195e-13, 1.0949e-13, 4.6927e-13, 7.4418e-14,\n -6.4044e-14, 3.2030e-15, -2.3706e-14, -1.5279e-13, 1.2198e-13,\n -9.0231e-14, 4.0310e-13, 2.9265e-14, 3.5740e-13, -9.0372e-14,\n -1.1759e-13, 1.0492e-13, 2.2165e-13, 4.9470e-14, -8.4463e-14,\n -2.7254e-14, 2.7537e-13, 2.3874e-13, 1.3377e-13, -2.7973e-13,\n 4.8388e-14, 1.5783e-13, 2.6616e-13, -4.4647e-14, -5.9064e-13,\n -1.4580e-13, -7.6228e-13, -1.3234e-13, -6.5695e-14, -2.3218e-13,\n -1.0695e-12, 2.0107e-14, 5.7095e-13, -7.1822e-13, 6.7514e-13,\n 3.1580e-13, 1.1580e-13, -3.3051e-14, -9.5876e-14, 1.0312e-13,\n 1.4046e-13, -4.4164e-14, 8.4496e-14, 4.9980e-15, -5.3139e-13,\n -5.8212e-14, 1.1469e-13, 1.3083e-13, 3.1538e-13, -5.0068e-15,\n 3.6533e-13, -2.2548e-13, 1.0224e-13, 4.2507e-13, 4.2845e-14,\n 7.9614e-14, 1.5780e-13, -1.2640e-13, -3.7255e-13, -4.6912e-13,\n -8.6663e-13, 5.0286e-14, 3.2264e-13, 3.2735e-13, 5.4790e-14,\n -8.9300e-13, -2.0957e-13, -3.7909e-13, -1.6328e-14, -1.1088e-13,\n -2.6458e-13, -3.3807e-13, 2.3972e-13, 4.0076e-13, 3.5796e-14,\n -1.0991e-15, -2.1516e-14, 1.1931e-14, 1.2181e-13, 3.3513e-14,\n -8.9033e-14, 3.1185e-13, -1.8132e-13, -3.1390e-13, 2.6686e-13,\n 7.9034e-14, -9.0151e-14, -3.2775e-13, 3.4320e-13, 6.1335e-14,\n -2.6142e-13, 5.8153e-14, 2.9827e-13, 5.3008e-13, -7.0381e-13,\n 3.9604e-13, -1.0920e-14, -3.5540e-14, 9.6239e-14, 6.5102e-14,\n 2.9372e-13, -1.9730e-13, -3.6673e-14, -3.6693e-14, 1.9663e-13,\n -1.0434e-13, -3.1688e-13, -7.0430e-13, -2.4103e-13, 1.3679e-13,\n -8.0641e-15, -3.6600e-14, -3.9293e-13, 4.8699e-14, -3.0263e-14,\n 4.9198e-14, 2.2137e-13, -1.1994e-13, 3.2443e-14, -1.4815e-14,\n 9.4273e-15, -3.0852e-13, -6.7853e-14, -3.2063e-13, -4.7419e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([6.8815e-08, 1.1431e-07, 6.5316e-08, 6.7323e-07, 2.3018e-07, 3.0344e-07,\n 1.0485e-08, 1.6033e-08, 6.2308e-08, 1.1847e-07, 1.5079e-08, 4.6543e-08,\n 4.1737e-06, 5.3364e-07, 4.4191e-07, 1.3915e-07, 4.1486e-07, 4.9672e-08,\n 5.9308e-07, 1.0714e-08, 4.1080e-07, 1.0211e-08, 6.1308e-08, 4.9559e-09,\n 8.2772e-08, 2.9019e-06, 3.5848e-07, 1.3312e-06, 1.0802e-07, 1.4352e-08,\n 2.3921e-06, 1.9566e-06, 3.0195e-08, 2.8263e-06, 1.2211e-06, 4.7461e-07,\n 1.0429e-06, 9.4918e-08, 1.3778e-06, 2.4904e-09, 1.3021e-08, 3.0346e-07,\n 1.0404e-06, 1.9434e-09, 3.5846e-08, 2.0902e-06, 2.3332e-06, 4.4351e-08,\n 1.6512e-08, 2.0164e-08, 6.9081e-08, 9.7847e-07, 1.1592e-07, 4.8812e-08,\n 8.5964e-07, 6.7240e-08, 8.4380e-07, 8.6663e-09, 6.6161e-07, 3.9582e-08,\n 6.0069e-08, 7.2053e-07, 1.6525e-08, 6.2184e-07, 2.0874e-08, 1.2298e-07,\n 2.3541e-08, 3.0208e-07, 1.4061e-06, 2.7530e-06, 3.0143e-06, 5.7346e-07,\n 2.0249e-08, 2.4367e-07, 3.2618e-07, 1.5807e-06, 9.0811e-09, 1.5061e-07,\n 3.5609e-07, 5.7961e-06, 1.1760e-07, 1.0766e-07, 2.8445e-08, 7.2781e-07,\n 2.7645e-07, 3.6817e-09, 3.2527e-09, 1.8198e-08, 2.6491e-07, 2.2500e-06,\n 2.0232e-07, 2.5255e-07, 6.8038e-09, 2.3341e-06, 1.8129e-08, 3.0810e-06,\n 1.7227e-07, 2.9338e-06, 8.6977e-07, 2.5659e-08, 2.1956e-08, 2.4785e-06,\n 5.6868e-09, 3.0930e-08, 6.9396e-07, 3.3027e-07, 1.7682e-09, 1.5470e-07,\n 5.9601e-08, 1.1332e-08, 1.3260e-08, 2.7958e-06, 8.2572e-08, 1.2929e-08,\n 4.7337e-08, 1.7282e-09, 8.8933e-09, 1.4180e-06, 1.2368e-08, 1.6475e-07,\n 2.0917e-06, 2.4207e-06, 2.6161e-07, 6.2951e-07, 6.4033e-08, 4.7492e-09,\n 1.0029e-07, 2.9765e-08, 1.4538e-08, 5.3627e-08, 1.2165e-08, 7.8130e-09,\n 4.0887e-08, 1.2620e-06, 3.5372e-07, 2.2126e-06, 3.5790e-09, 6.7814e-07,\n 6.1752e-07, 4.0368e-08, 1.3801e-07, 3.0419e-07, 2.0752e-08, 3.6252e-08,\n 4.6614e-09, 5.7660e-08, 1.1702e-08, 6.5631e-06, 9.6321e-07, 3.2645e-08,\n 3.6761e-08, 2.1097e-08, 3.9442e-08, 6.3753e-07, 1.1087e-08, 7.6223e-07,\n 3.0663e-09, 3.5391e-07, 4.1106e-09, 8.4157e-08, 1.1085e-06, 2.5312e-08,\n 3.7461e-07, 3.1616e-09, 1.0736e-08, 1.3643e-08, 5.4205e-07, 3.3889e-06,\n 1.4243e-07, 4.0177e-08, 6.5613e-09, 8.0964e-08, 3.0212e-09, 1.8981e-07,\n 2.8467e-06, 1.6690e-08, 3.2157e-09, 3.2150e-06, 3.4114e-09, 1.3576e-07,\n 5.2517e-08, 3.5964e-07, 5.0491e-08, 9.0387e-07, 5.0774e-07, 2.2634e-08,\n 9.3280e-07, 1.2799e-06, 4.5522e-09, 2.4767e-07, 1.4238e-06, 4.4580e-08,\n 1.3230e-06, 6.9634e-08, 2.2728e-06, 3.7328e-07, 3.4703e-09, 1.4746e-07,\n 4.2392e-06, 1.7912e-07, 1.2074e-08, 8.5857e-07, 5.0202e-08, 2.2912e-07,\n 7.1778e-09, 7.1160e-07, 2.6014e-06, 1.7161e-08, 2.6253e-07, 5.7052e-07,\n 1.7860e-06, 2.2281e-07, 1.2249e-07, 1.4879e-06, 5.4954e-07, 2.9576e-08,\n 9.3524e-08, 5.9262e-09, 4.9508e-07, 1.4616e-07, 2.3134e-07, 5.1200e-08,\n 1.8116e-06, 5.1642e-07, 1.2785e-09, 1.6568e-08, 3.1703e-06, 2.1529e-06,\n 9.4679e-07, 2.1046e-09, 3.2685e-07, 5.2324e-08, 9.0146e-08, 1.3547e-07,\n 2.5302e-07, 5.2440e-06, 3.3386e-08, 9.9570e-09, 1.2192e-07, 4.3965e-08,\n 1.0125e-07, 3.3358e-09, 7.1735e-07, 6.4511e-08, 3.8251e-08, 3.0559e-08,\n 2.2534e-08, 1.7812e-06, 6.6288e-08, 5.5168e-08, 5.2011e-07, 3.7425e-08,\n 2.2150e-07, 2.2790e-06, 1.0699e-06, 3.4083e-06, 1.5989e-09, 1.8434e-07,\n 4.1769e-08, 1.5759e-08, 1.0835e-08, 4.1049e-08, 2.0794e-08, 3.6149e-07,\n 1.3447e-06, 9.3981e-08, 3.5667e-08, 1.2482e-07, 8.3703e-08, 2.2424e-06,\n 4.1649e-07, 1.4788e-08, 7.7860e-07, 3.3117e-07, 5.7945e-08, 8.0672e-09,\n 4.2983e-08, 5.3274e-08, 4.7607e-07, 1.3127e-06, 4.6556e-06, 7.6240e-07,\n 2.9260e-07, 2.5162e-06, 8.7332e-08, 2.1109e-08, 2.7045e-07, 1.3711e-06,\n 3.4264e-06, 7.4612e-09, 1.3189e-07, 1.2354e-08, 3.0736e-06, 1.0089e-08,\n 2.3882e-07, 1.6277e-06, 2.6259e-08, 1.2527e-07, 3.6507e-07, 8.7926e-08,\n 1.2912e-08, 3.2410e-09, 3.9973e-06, 1.1843e-06, 1.0810e-07, 4.2479e-09,\n 5.3244e-08, 1.3373e-06, 7.9746e-09, 5.0283e-09, 2.1570e-08, 2.2572e-06,\n 2.5677e-07, 4.9728e-06, 3.1080e-06, 2.7929e-09, 1.0613e-08, 1.5949e-06,\n 1.5283e-06, 3.0802e-06, 3.4524e-09, 6.7190e-09, 1.4096e-07, 2.1875e-07,\n 1.2246e-07, 5.2571e-09, 3.2005e-07, 5.5726e-08, 7.0693e-09, 3.6083e-06,\n 1.0651e-06, 6.5431e-08, 1.7785e-06, 2.7655e-07, 2.2874e-06, 4.4833e-08,\n 6.7841e-08, 2.7117e-08, 5.8905e-07, 5.0237e-08, 1.4377e-06, 6.3503e-08,\n 6.1788e-08, 4.3620e-07, 2.7271e-06, 6.6454e-08, 2.5875e-08, 2.4713e-08,\n 8.1114e-07, 1.3960e-07, 5.6782e-09, 2.8261e-07, 9.1110e-09, 1.4137e-07,\n 1.0819e-07, 7.5919e-09, 3.8845e-06, 3.4092e-09, 1.2037e-08, 6.7064e-07,\n 3.9827e-07, 1.9859e-08, 4.8680e-08, 1.5128e-06, 1.0115e-07, 8.1017e-07,\n 2.3308e-08, 1.3703e-06, 4.6235e-06, 2.1056e-06, 4.8178e-07, 5.7603e-09,\n 1.5135e-07, 7.6621e-08, 5.2682e-09, 1.6534e-07, 3.6890e-08, 2.1660e-06,\n 1.8778e-06, 3.7638e-06, 5.8125e-08, 1.3026e-08, 2.4552e-08, 2.3376e-07,\n 3.6013e-09, 1.3303e-07, 6.9594e-08, 3.3379e-08, 8.6248e-07, 5.0584e-07,\n 2.5315e-08, 6.8476e-08, 1.9966e-06, 3.0183e-07, 1.4865e-06, 9.9354e-08,\n 1.9673e-06, 1.3524e-07, 6.3678e-08, 5.4300e-06, 2.0296e-08, 1.0109e-07,\n 1.3339e-08, 3.1786e-07, 4.2612e-08, 1.0458e-06, 3.8518e-07, 3.4820e-07,\n 9.1578e-06, 1.9269e-09, 4.6820e-08, 2.0771e-07, 7.3877e-07, 5.0900e-08,\n 7.6306e-08, 1.4667e-07, 9.6423e-08, 1.6577e-08, 3.8424e-07, 9.8493e-09,\n 8.1914e-09, 6.3228e-07, 2.9268e-06, 1.0956e-07, 4.8254e-06, 1.7637e-06,\n 1.0470e-06, 3.4809e-06, 4.8400e-09, 1.5664e-07, 1.0565e-07, 2.8326e-08,\n 5.0240e-07, 5.4376e-07, 1.7899e-07, 2.3502e-08, 1.5700e-06, 5.7369e-08,\n 1.0311e-06, 6.5276e-08, 4.6816e-07, 2.2221e-08, 7.3120e-08, 3.3988e-06,\n 5.0370e-07, 1.8475e-06, 4.7951e-06, 7.0456e-07, 4.0245e-07, 2.1371e-07,\n 2.8554e-07, 3.3177e-07, 2.5210e-09, 1.4885e-07, 5.0445e-09, 2.1762e-08,\n 1.8939e-06, 2.5892e-07, 3.4707e-07, 9.1868e-09, 4.4063e-06, 6.7830e-07,\n 1.8265e-06, 3.5558e-07, 5.9835e-09, 5.0896e-07, 4.2609e-07, 4.0778e-08,\n 3.2506e-08, 2.2404e-07, 2.4399e-09, 1.6439e-06, 8.8815e-08, 1.1124e-08,\n 3.6539e-08, 5.5108e-07, 1.2937e-08, 2.2683e-08, 6.0967e-09, 3.2012e-07,\n 1.6736e-08, 3.1766e-09, 7.7976e-09, 1.5151e-07, 1.1922e-08, 3.7948e-06,\n 9.4855e-07, 1.8161e-06, 3.1972e-09, 9.9103e-08, 3.1150e-08, 3.1024e-06,\n 2.7070e-08, 1.2024e-08, 3.0176e-08, 5.0474e-08, 1.7711e-08, 4.6368e-07,\n 5.8780e-09, 1.0035e-07, 5.7409e-07, 3.6810e-06, 4.2245e-08, 3.0264e-08,\n 1.2776e-08, 9.3281e-09, 1.3789e-07, 1.5498e-08, 6.8000e-08, 1.0984e-06,\n 2.4728e-08, 9.0117e-09, 2.0710e-08, 1.1193e-06, 1.5015e-07, 3.1654e-07,\n 4.7941e-09, 4.5815e-07, 1.0470e-06, 5.4123e-09, 6.7260e-06, 4.9118e-07,\n 1.0955e-07, 6.6010e-08, 1.0033e-08, 2.4970e-08, 2.1799e-08, 3.6986e-06,\n 1.3020e-07, 3.7494e-08, 2.7601e-09, 9.0987e-08, 2.1780e-06, 1.2566e-08,\n 2.1468e-06, 1.7319e-07, 1.2823e-06, 8.9644e-07, 7.7715e-08, 2.6337e-07,\n 1.0780e-08, 4.9143e-08, 2.5508e-06, 7.1602e-08, 9.4578e-09, 4.2469e-08,\n 1.6518e-09, 3.0128e-06, 1.8080e-06, 1.3780e-07, 2.7937e-08, 1.0774e-06,\n 7.1713e-09, 1.6578e-07, 1.0906e-07, 1.9535e-07, 2.8906e-09, 4.5886e-09,\n 3.5490e-09, 5.8857e-07, 4.1676e-07, 1.3914e-06, 4.6961e-09, 7.8047e-07,\n 1.3610e-06, 3.0314e-07, 3.7616e-07, 3.3249e-08, 4.9009e-07, 2.6077e-06,\n 1.4994e-08, 6.0555e-09, 2.2689e-08, 2.4316e-06, 1.0475e-05, 4.0972e-07,\n 2.0783e-07, 1.1614e-07, 7.7379e-09, 9.4854e-08, 1.7512e-08, 1.3610e-07,\n 2.8317e-06, 5.5385e-09, 1.0441e-06, 2.2816e-06, 9.5673e-08, 5.2225e-08,\n 1.5595e-06, 1.9355e-07, 4.0186e-08, 1.1470e-06, 2.6447e-09, 2.6374e-09,\n 8.0690e-09, 1.7331e-08, 1.2153e-07, 1.9681e-07, 1.4155e-08, 2.0893e-07,\n 1.1604e-06, 6.5879e-07, 4.8576e-08, 3.9625e-09, 2.9204e-07, 1.4611e-08,\n 2.6693e-07, 8.5194e-07, 9.9829e-07, 3.7208e-06, 2.6199e-09, 2.3780e-09,\n 3.9328e-07, 2.3761e-06, 7.6608e-08, 1.0164e-08, 2.5252e-06, 9.9282e-09,\n 3.4572e-09, 2.0161e-08, 1.0999e-07, 8.8428e-08, 8.3757e-08, 2.6806e-08,\n 4.9752e-07, 4.9754e-08, 5.4083e-08, 2.3402e-06, 7.6579e-08, 5.2979e-06,\n 1.2436e-07, 3.6974e-07, 6.4952e-07, 3.1474e-08, 3.2129e-06, 6.8173e-07,\n 8.4593e-07, 5.0486e-09, 5.3100e-09, 1.8336e-08], device='cuda:0')" + }, + "40": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.5456e-16, -8.4836e-14, 1.2869e-14, 3.7091e-15, 1.0116e-16,\n -1.3298e-14, -1.2598e-15, -2.5790e-16, -3.6864e-16, -8.1806e-15,\n -1.3096e-15, -4.4100e-17, 1.3174e-15, -2.3933e-15, -9.2009e-15,\n -1.4000e-14, -3.8786e-15, -6.3602e-17, 3.1347e-15, 8.1652e-16,\n -3.7362e-15, -1.0441e-14, -6.1562e-16, -2.3359e-16, -1.1039e-15,\n -3.8903e-15, -9.2221e-16, 4.7097e-18, 7.1039e-16, -6.4829e-16,\n 6.5253e-15, 9.5199e-16, 6.0681e-16, -1.0212e-14, 3.2453e-16,\n -2.3551e-14, -6.5135e-15, 4.1303e-15, -8.7617e-15, -1.9157e-16,\n 9.4851e-16, -8.3999e-16, 3.0805e-16, -2.6847e-15, -1.6363e-14,\n -2.4529e-15, -1.1698e-14, 1.1802e-15, -4.6472e-16, 6.1020e-15,\n -2.4134e-15, -2.4429e-18, 9.5633e-16, 1.0305e-15, -1.2724e-16,\n 3.0073e-15, -5.0868e-16, 6.4383e-16, 1.5142e-15, 8.3782e-16,\n -1.9519e-15, -1.5627e-14, -1.4924e-15, 6.0643e-16, -3.1041e-15,\n -7.4078e-15, -2.2667e-15, 1.3060e-15, -9.6384e-16, 4.4337e-15,\n 1.5477e-16, 1.2567e-14, -1.7562e-16, -1.7608e-16, -5.1538e-17,\n -1.8685e-15, -3.8597e-15, 2.4940e-16, -8.0000e-15, 4.3118e-15,\n -1.4495e-15, -1.0113e-14, 4.5197e-16, -2.6038e-16, -2.9244e-16,\n 6.8922e-16, 1.6588e-15, -2.0129e-14, 3.2536e-16, 1.9592e-15,\n 2.0777e-16, 3.7468e-17, -4.7400e-15, 1.6753e-15, -2.8625e-14,\n 7.8409e-16, -2.5572e-14, -1.4668e-14, 1.2880e-15, -7.6386e-16,\n -1.1702e-14, -5.6984e-15, 5.9376e-16, -1.9424e-14, -2.7003e-15,\n -3.3076e-15, -3.6145e-16, 2.1269e-14, -5.5399e-16, -1.2618e-14,\n -4.7747e-15, -4.6453e-15, 5.6706e-16, -8.5552e-16, -1.1213e-15,\n 1.6849e-16, 2.2096e-16, -1.0767e-15, -1.7319e-15, 1.0172e-16,\n -5.0671e-17, -4.5404e-16, -3.1870e-17, -4.5170e-15, 3.6321e-16,\n 4.5342e-16, -3.6782e-15, -1.8513e-15, 1.2607e-15, -2.6350e-14,\n -1.5788e-15, -3.7828e-15, -3.8873e-17, -2.7786e-15, -8.3156e-16,\n -4.0989e-15, 5.1605e-16, 2.2468e-14, 1.0175e-15, 2.1342e-16,\n -2.8605e-15, -5.1758e-14, 9.5087e-17, -2.0775e-15, -2.3931e-17,\n -4.6766e-16, 1.4440e-15, -7.9387e-15, 3.1423e-16, -4.7402e-16,\n -3.6871e-16, 6.1425e-16, -1.1325e-16, -2.6488e-15, -4.1399e-16,\n -4.5251e-15, -4.5142e-16, 1.3151e-15, 1.1377e-16, -4.6730e-16,\n -4.1490e-15, -8.9821e-15, -6.4017e-16, -2.4566e-15, -4.8736e-15,\n 1.6365e-15, -2.1370e-16, -2.4760e-15, 4.0781e-15, -8.9235e-17,\n 3.3752e-15, -5.9157e-16, -1.5885e-14, -6.7666e-16, -5.0287e-15,\n -1.5581e-15, -8.1050e-16, -1.7588e-14, -3.0694e-15, -5.2190e-15,\n -1.1077e-14, -4.8001e-16, -2.7106e-14, -7.5230e-18, -3.9101e-14,\n 6.1290e-16, 2.4567e-15, -8.0274e-16, -7.0303e-16, -1.8896e-15,\n 5.6089e-16, -1.8211e-14, -1.2493e-14, -4.8996e-17, -1.3710e-15,\n -3.4382e-15, 4.7041e-16, -7.1974e-16, -1.0571e-15, -1.4353e-15,\n 2.1764e-15, -4.5155e-16, 1.5480e-16, -1.2339e-15, -4.2644e-18,\n 2.9854e-15, -6.2270e-15, -5.4420e-15, -2.5855e-16, -4.9832e-15,\n -6.4398e-15, -1.7563e-15, -9.1685e-15, -1.2299e-15, 5.9170e-17,\n -7.3218e-16, -1.2878e-14, -2.1160e-16, 8.9436e-16, 4.7808e-15,\n -6.9757e-15, 3.8125e-16, -5.1543e-15, 2.0434e-15, -2.9826e-16,\n -2.4349e-15, 4.6993e-15, -9.8821e-15, -2.0326e-16, -9.1055e-16,\n -2.1424e-14, 3.8864e-16, -3.3174e-16, 7.8021e-16, -5.6902e-16,\n 9.5878e-16, -7.0463e-16, -2.5673e-17, -1.2409e-14, -4.8172e-16,\n 5.1422e-16, -6.3181e-18, -9.0682e-15, -1.7851e-14, 3.5736e-15,\n 9.0027e-17, 1.1472e-15, -3.2549e-16, 8.6598e-16, -2.5018e-16,\n -1.7041e-14, 1.4008e-15, 1.7160e-16, 1.4656e-15, -3.1752e-16,\n -6.7959e-15, -2.9940e-15, 6.5273e-15, -1.4854e-15, 2.7764e-16,\n 4.1892e-16, 1.4453e-15, 2.4916e-16, -3.2708e-15, -4.4258e-15,\n -1.1193e-15, 1.3947e-16, -2.7538e-14, -5.8326e-16, -1.1717e-14,\n -3.6210e-16, -1.6775e-15, -1.0303e-15, -3.4014e-14, -2.3955e-16,\n -4.0934e-15, -2.4506e-14, -4.4004e-15, 6.5751e-15, -9.2412e-15,\n 4.5885e-15, 4.1284e-15, -2.8667e-16, -7.5961e-15, -6.9419e-17,\n -1.1083e-14, -7.0842e-15, -6.5042e-15, -5.4284e-16, 5.5344e-17,\n 1.4317e-15, -1.6755e-16, -2.1329e-14, 9.2252e-15, 6.1447e-16,\n 6.2620e-15, -5.3444e-15, 3.9937e-16, -4.0757e-15, -1.3359e-15,\n 7.0559e-15, -2.0696e-15, -7.0227e-16, -6.9944e-15, -4.0369e-15,\n 2.9192e-15, 4.5475e-16, -6.8780e-16, -3.2677e-14, -1.9564e-14,\n 1.0124e-15, -2.0650e-15, 2.3068e-16, 4.6613e-15, -4.1736e-16,\n -1.0554e-14, -2.7310e-15, 8.2694e-15, -4.4249e-15, -7.1004e-16,\n 4.7731e-16, 3.1755e-15, 1.5984e-16, -9.4062e-16, 8.4972e-16,\n -2.5568e-15, 3.2355e-16, 1.3049e-15, 2.2307e-16, 2.2430e-17,\n -1.3836e-14, -2.5205e-15, -1.1856e-15, 7.5326e-16, -2.3524e-15,\n 1.8292e-15, 1.6454e-15, -1.6247e-15, 1.1952e-16, -9.3706e-15,\n 1.2519e-15, 5.8667e-15, -2.4094e-15, -9.8273e-16, -4.6154e-14,\n -1.1867e-14, 4.9265e-16, 4.1033e-16, -1.0980e-14, -6.1564e-16,\n 5.0135e-15, 1.3319e-15, -5.4058e-16, -3.6435e-15, -1.5068e-15,\n -1.1350e-14, -2.0572e-14, 1.7941e-15, -1.6099e-14, -1.3220e-15,\n -2.3923e-16, -4.1206e-15, 5.0407e-17, -3.8848e-14, -3.0407e-15,\n -3.9091e-16, -2.4879e-14, -6.9824e-15, -8.6529e-15, -1.2510e-14,\n 2.9287e-16, -2.2217e-16, -1.1915e-14, -1.4755e-16, -2.9663e-16,\n -8.1375e-16, -1.6476e-16, -5.7174e-15, -3.4766e-16, -1.3147e-15,\n 1.5120e-15, -1.9972e-16, 4.0233e-16, 3.5378e-15, -3.3169e-16,\n -3.6173e-16, -2.1466e-15, 1.6639e-16, -3.9137e-16, -3.9681e-15,\n 9.3208e-16, -2.9114e-16, -1.5635e-14, -2.6841e-16, -2.4454e-14,\n 1.9152e-16, -1.8276e-14, -3.6145e-14, -4.2504e-15, -1.9133e-15,\n -2.2264e-14, 1.2228e-15, -1.3352e-15, -2.4724e-15, -7.5903e-15,\n -1.1365e-14, 1.5063e-15, -1.6872e-14, -2.9155e-15, 1.6583e-15,\n 8.1267e-15, 3.6265e-15, -3.8833e-14, -2.0932e-16, -8.1373e-15,\n 1.6413e-15, 1.3981e-16, 2.3034e-15, -1.0497e-14, -3.3030e-14,\n 3.2678e-16, 1.8304e-15, -2.4818e-15, -1.4716e-15, -7.8960e-15,\n 3.2622e-15, -1.0922e-14, -5.0567e-16, 1.4396e-15, 1.0697e-15,\n -1.3462e-14, 1.4919e-16, -1.3111e-15, -1.2872e-16, 1.1691e-15,\n -4.3684e-15, -2.7882e-15, -1.8333e-15, -5.1043e-16, 2.0542e-15,\n 3.0950e-15, -5.1744e-16, -1.8147e-15, 9.8149e-16, -9.0301e-17,\n -3.2931e-15, -3.8489e-14, -3.1698e-16, -3.9988e-15, -6.7199e-15,\n -2.6066e-14, 6.8476e-16, 4.2494e-16, -2.0378e-15, 3.4818e-14,\n -9.7478e-15, -2.8755e-15, -1.0801e-14, 8.9617e-16, -6.3693e-15,\n 7.5690e-16, -1.8277e-15, -4.4016e-14, -4.5757e-15, -1.2786e-14,\n 6.4339e-17, 1.1298e-16, -6.6184e-14, -1.0859e-14, -4.0858e-16,\n 1.1586e-15, 4.3925e-16, -1.4761e-15, -3.7681e-16, 1.7248e-16,\n -5.1738e-16, 4.9595e-15, 9.7643e-17, -1.1588e-16, 2.4593e-16,\n 2.3598e-16, 1.5418e-16, -6.5600e-15, 2.3810e-16, -2.4798e-15,\n -1.5202e-14, -3.1882e-16, -4.1652e-15, 2.9464e-16, 1.2496e-15,\n 2.1121e-16, -5.5674e-16, 5.0580e-17, -3.4918e-16, -3.5503e-15,\n -8.9568e-15, -3.5178e-16, 8.5620e-16, 1.2091e-15, -1.9175e-16,\n -3.6079e-15, -3.4945e-15, -7.6821e-16, 4.0337e-16, -8.9255e-16,\n -6.8451e-16, -2.7354e-15, 5.1599e-16, 1.6998e-14, -1.6524e-15,\n -3.0160e-15, 3.1238e-16, -6.1277e-16, 6.5099e-16, -1.8213e-15,\n 3.4150e-15, -4.6537e-16, -2.3182e-15, -2.1118e-14, 2.4555e-16,\n -1.3926e-14, -1.6204e-15, 1.9289e-15, 6.3295e-16, 2.6141e-15,\n -1.6127e-14, -4.5389e-16, -3.4624e-15, 5.7123e-16, -3.6090e-15,\n -2.7719e-15, -4.9195e-16, 3.7424e-15, 3.1585e-16, -1.9082e-14,\n -1.0961e-14, -2.5279e-16, -1.4135e-15, -1.1288e-14, -5.4953e-15,\n -3.6091e-17, -2.0169e-14, -3.3931e-15, -4.7818e-16, 3.9202e-16,\n -3.1463e-14, -3.4276e-16, 4.5041e-15, -1.1920e-14, 5.8332e-15,\n -3.4765e-15, -5.6343e-17, -1.7210e-15, -5.1053e-15, -2.9903e-15,\n -3.6942e-16, -1.4166e-15, 1.0206e-15, 1.0348e-15, -3.5662e-14,\n -6.6371e-15, -7.5113e-16, 3.8567e-16, 2.6369e-16, 9.2106e-16,\n 1.0294e-15, 1.7343e-15, -4.3128e-15, 3.1318e-15, 5.3859e-16,\n 1.2805e-16, -7.1737e-16, 5.7629e-16, -7.8695e-15, -1.3509e-14,\n -2.8455e-14, 4.4271e-17, 6.2362e-16, -7.0318e-16, -7.6321e-16,\n -3.1781e-14, -1.6983e-14, 3.0921e-16, -1.2826e-15, -4.5403e-15,\n -1.7992e-14, -6.2554e-15, -5.3982e-15, 1.8343e-16, -1.0515e-16,\n -5.9664e-16, -1.2925e-15, -2.1511e-15, -6.7027e-16, -1.1473e-15,\n -2.3397e-15, 3.9971e-16, -1.0562e-14, -8.8856e-15, 1.3987e-15,\n -2.4102e-16, -5.1985e-15, -2.0935e-14, 1.8742e-16, 9.4877e-16,\n -3.1015e-17, -7.2124e-15, 1.9283e-15, 4.1695e-15, -1.4749e-14,\n 1.5606e-16, -2.8710e-16, 5.0818e-16, -1.0787e-16, -3.3865e-15,\n -1.7429e-14, -1.1858e-14, -3.5554e-15, 1.0058e-15, -8.0017e-16,\n -1.0647e-15, -1.3851e-14, -1.0281e-13, -6.7894e-14, -6.0785e-17,\n -2.1638e-15, -2.1282e-16, -6.1626e-15, -3.3018e-16, -1.9267e-15,\n -1.0084e-16, -4.9508e-16, -5.2314e-15, -3.4064e-15, -1.1091e-14,\n -6.8841e-15, -3.9837e-15, -1.7137e-15, -1.0314e-16, 4.3435e-17],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.9609e-11, 4.3088e-12, 1.4178e-12, 6.6524e-11, 2.3270e-11, 8.0466e-11,\n 1.1727e-11, 1.9239e-11, 6.3580e-12, 7.7326e-12, 2.2493e-12, 9.4451e-12,\n 7.3195e-10, 4.3003e-10, 2.4359e-11, 2.8320e-11, 7.9829e-12, 1.1486e-11,\n 3.8635e-11, 2.4150e-13, 6.5119e-11, 3.6327e-13, 7.1933e-12, 2.3657e-13,\n 5.3568e-13, 4.9146e-10, 5.5976e-12, 7.8536e-11, 9.5611e-13, 2.6004e-13,\n 1.4233e-10, 2.4202e-10, 6.5333e-12, 2.2360e-10, 1.4282e-11, 4.2600e-10,\n 2.2980e-10, 1.1102e-10, 7.3645e-10, 1.0375e-13, 1.4784e-13, 3.9865e-12,\n 4.7372e-10, 5.6093e-13, 3.9743e-12, 1.2679e-09, 9.5181e-10, 1.1467e-12,\n 1.8307e-13, 7.6854e-13, 7.9504e-13, 1.0332e-11, 1.8603e-11, 2.1588e-11,\n 3.4325e-11, 1.3022e-12, 3.0246e-11, 6.3036e-13, 1.0723e-11, 5.2099e-13,\n 6.5907e-13, 1.1703e-10, 3.7488e-12, 3.9056e-11, 3.0992e-12, 2.3882e-12,\n 3.9802e-12, 1.9988e-12, 1.2626e-10, 1.1127e-09, 5.4670e-10, 6.5300e-11,\n 3.2075e-12, 1.3956e-11, 2.3230e-11, 3.7739e-10, 8.8419e-12, 6.7465e-13,\n 1.3633e-10, 2.7405e-09, 1.5796e-11, 9.6609e-12, 2.6317e-13, 2.5701e-11,\n 5.1239e-12, 2.6327e-14, 7.9505e-14, 4.6832e-12, 1.4261e-12, 1.0347e-09,\n 1.9614e-11, 2.7326e-11, 6.2592e-12, 1.0500e-10, 2.5122e-12, 9.9585e-10,\n 2.5721e-11, 3.1519e-10, 3.4475e-10, 4.1455e-13, 3.7004e-12, 2.0878e-09,\n 3.0315e-13, 2.9769e-12, 5.3910e-11, 7.2148e-12, 4.2690e-14, 4.8183e-12,\n 3.4859e-13, 3.9697e-13, 1.2351e-11, 1.1857e-09, 1.2778e-12, 1.4185e-12,\n 4.4577e-12, 3.0955e-13, 8.9233e-14, 7.2152e-12, 6.1704e-13, 6.9349e-12,\n 4.1045e-10, 2.0905e-09, 1.0112e-11, 3.3274e-10, 6.9766e-13, 3.2967e-13,\n 1.7112e-12, 1.4201e-12, 9.2319e-13, 3.1118e-12, 1.7510e-12, 1.8414e-13,\n 1.4711e-12, 1.9941e-10, 2.5970e-12, 2.1869e-10, 8.7382e-14, 8.2154e-10,\n 1.1107e-11, 8.8329e-12, 9.5698e-13, 1.9029e-10, 4.0804e-13, 8.0010e-12,\n 2.0220e-13, 3.6790e-12, 5.2041e-13, 1.4840e-09, 1.3220e-10, 6.2553e-12,\n 5.1857e-13, 1.2890e-12, 8.3915e-12, 1.0602e-10, 9.6561e-13, 6.7763e-12,\n 6.2517e-14, 1.3528e-12, 3.5601e-14, 2.9494e-12, 1.0059e-11, 9.3723e-13,\n 4.2496e-11, 1.0104e-12, 5.4892e-13, 5.1567e-13, 3.7702e-11, 2.7635e-10,\n 1.2644e-11, 9.0050e-12, 8.5245e-14, 1.9304e-11, 9.3629e-13, 2.1559e-12,\n 1.4804e-09, 8.4974e-13, 1.6323e-13, 1.0052e-09, 2.5908e-14, 6.3497e-13,\n 1.5923e-12, 2.1608e-10, 5.3248e-13, 1.5385e-10, 2.0134e-10, 1.5173e-13,\n 6.3242e-10, 2.0442e-10, 4.3524e-12, 7.6067e-11, 5.1768e-11, 1.3642e-11,\n 2.8802e-11, 1.8112e-12, 4.1900e-11, 1.0545e-11, 3.1314e-13, 2.4028e-11,\n 6.0238e-10, 1.6472e-12, 1.5471e-13, 1.0413e-10, 1.2896e-11, 9.1491e-12,\n 3.2268e-12, 1.0907e-11, 9.4719e-11, 2.5219e-12, 3.7455e-12, 4.1321e-12,\n 6.3255e-10, 1.6503e-12, 3.7553e-12, 7.9813e-10, 3.7234e-12, 5.1557e-13,\n 5.2534e-12, 1.2818e-12, 2.6149e-11, 4.8907e-11, 1.2365e-10, 1.9687e-12,\n 5.3977e-11, 2.5625e-10, 2.7459e-12, 9.8731e-13, 4.3242e-10, 2.8686e-10,\n 4.8075e-10, 3.2383e-13, 2.0161e-11, 2.7070e-11, 1.2788e-12, 1.4007e-12,\n 3.0281e-12, 1.9419e-10, 2.2425e-13, 9.6602e-13, 1.1390e-11, 2.8698e-13,\n 1.9902e-13, 2.9939e-12, 5.5950e-12, 9.8845e-12, 3.5497e-13, 5.5716e-13,\n 6.1364e-13, 4.7173e-11, 2.9555e-13, 7.2261e-13, 6.3927e-12, 8.4014e-12,\n 6.6063e-11, 2.3585e-10, 1.3775e-10, 2.5916e-09, 8.5323e-13, 1.2004e-11,\n 5.5948e-12, 1.6285e-13, 1.7885e-13, 4.6976e-12, 4.2165e-13, 1.9391e-11,\n 5.0378e-11, 6.5024e-13, 1.3810e-13, 7.4001e-12, 2.7852e-11, 8.8911e-11,\n 2.8206e-12, 1.9277e-12, 4.4622e-10, 3.5490e-10, 1.8334e-11, 9.2616e-12,\n 5.9337e-12, 7.4907e-12, 9.8510e-12, 6.4045e-11, 2.8270e-10, 5.2497e-12,\n 4.5215e-12, 4.3540e-10, 2.0256e-12, 3.2770e-12, 5.8711e-11, 3.0726e-10,\n 1.4165e-09, 5.0643e-14, 9.8491e-13, 1.3450e-11, 9.6892e-10, 2.1551e-13,\n 5.8651e-11, 3.8498e-10, 3.2901e-11, 1.0955e-12, 4.1582e-11, 2.3267e-12,\n 2.8406e-13, 1.1126e-12, 8.2914e-10, 2.6277e-11, 3.9971e-13, 6.8757e-14,\n 1.1520e-12, 2.7241e-11, 1.7036e-13, 1.0146e-13, 3.8791e-13, 5.1286e-10,\n 2.3064e-12, 1.6704e-09, 2.3344e-10, 1.7970e-13, 2.3318e-13, 1.0223e-09,\n 6.1831e-11, 2.1740e-10, 1.2089e-13, 2.4812e-12, 3.5507e-11, 4.8865e-13,\n 9.1620e-13, 3.4957e-13, 7.6325e-12, 7.0599e-13, 1.2403e-13, 4.2419e-10,\n 3.8678e-10, 2.1720e-11, 5.3556e-10, 4.5978e-12, 2.6265e-10, 2.3238e-11,\n 2.4063e-11, 3.5008e-12, 4.3135e-12, 1.3870e-12, 9.6037e-11, 6.7281e-13,\n 3.5083e-13, 9.1242e-12, 1.4192e-09, 5.1040e-13, 5.3735e-12, 3.7973e-13,\n 1.2352e-11, 1.0485e-12, 1.9703e-12, 8.7115e-12, 1.2205e-13, 1.3264e-10,\n 3.5722e-11, 1.1675e-12, 1.9654e-09, 4.2267e-12, 4.2486e-13, 1.3085e-11,\n 2.1691e-12, 3.4358e-12, 7.0398e-13, 4.7263e-10, 1.8915e-12, 1.1730e-11,\n 1.1005e-12, 8.6519e-10, 1.3439e-09, 3.7827e-10, 3.8754e-12, 1.8068e-13,\n 3.3493e-12, 1.1716e-12, 3.1683e-13, 1.6249e-12, 7.7701e-13, 2.0149e-09,\n 2.7320e-10, 1.9618e-10, 4.0254e-13, 1.5794e-12, 4.3480e-12, 3.8137e-11,\n 9.0077e-14, 7.8185e-13, 2.1595e-13, 1.9513e-13, 1.1843e-11, 1.2580e-10,\n 4.2809e-12, 2.3204e-11, 4.9027e-11, 2.9934e-12, 4.1397e-10, 5.1780e-11,\n 1.5999e-09, 1.6887e-12, 4.5171e-11, 1.4241e-09, 2.5093e-13, 6.4010e-13,\n 7.0788e-12, 6.6997e-12, 1.7078e-12, 3.1008e-10, 5.7097e-11, 1.1981e-10,\n 2.8384e-09, 5.5745e-14, 7.3367e-12, 3.9257e-11, 2.0798e-10, 2.3031e-11,\n 3.3895e-13, 1.0450e-12, 2.1298e-12, 2.0490e-12, 2.8008e-12, 5.8056e-13,\n 6.9823e-13, 2.5012e-11, 1.4326e-09, 4.5930e-12, 2.4541e-09, 3.0003e-10,\n 9.1866e-10, 1.1700e-09, 1.0429e-12, 1.4738e-12, 4.1467e-12, 4.6417e-13,\n 4.4487e-11, 7.6243e-12, 5.5308e-11, 9.0510e-13, 6.2435e-10, 6.0312e-12,\n 3.8151e-10, 1.0463e-10, 1.4460e-10, 4.7371e-12, 2.7553e-12, 2.8084e-10,\n 2.5319e-11, 1.4947e-10, 4.9635e-09, 2.8656e-11, 1.8635e-11, 7.3475e-12,\n 1.1464e-10, 7.8001e-12, 9.2427e-14, 2.1727e-11, 4.2709e-13, 2.6733e-12,\n 3.2783e-10, 3.4048e-12, 1.3358e-11, 1.0466e-12, 2.5969e-09, 4.7233e-10,\n 1.9231e-10, 1.5979e-11, 2.8996e-12, 1.1635e-10, 1.1258e-11, 7.9776e-12,\n 6.3090e-12, 4.9420e-12, 1.5262e-13, 1.6251e-10, 2.7844e-13, 6.9485e-13,\n 5.3743e-13, 8.2330e-12, 3.9270e-12, 1.5870e-12, 3.1610e-13, 2.6687e-12,\n 2.0503e-12, 2.1062e-12, 1.4745e-13, 2.3892e-12, 1.5314e-12, 2.7726e-09,\n 2.0990e-11, 7.4674e-10, 5.5248e-13, 7.9790e-13, 9.3450e-13, 3.2737e-10,\n 3.4874e-13, 2.0120e-13, 4.9485e-12, 3.3316e-12, 2.0913e-12, 2.2374e-12,\n 4.1201e-13, 9.3758e-11, 6.4296e-12, 5.7828e-10, 1.4466e-12, 6.0091e-12,\n 1.4697e-12, 1.9448e-13, 1.9466e-11, 1.8626e-11, 5.7082e-11, 2.2550e-10,\n 3.3027e-12, 3.0897e-13, 2.8677e-12, 1.4070e-10, 5.5244e-11, 1.6130e-12,\n 4.6164e-13, 2.8430e-11, 7.1222e-12, 1.3864e-13, 3.5898e-09, 2.0780e-10,\n 3.8907e-11, 1.3260e-11, 6.8328e-14, 5.2993e-12, 6.0266e-12, 1.0481e-09,\n 6.7110e-13, 1.7913e-12, 2.5155e-12, 2.3731e-12, 4.9149e-10, 2.7571e-13,\n 6.3269e-10, 1.4996e-12, 6.2655e-11, 3.4751e-12, 8.7622e-13, 3.2887e-12,\n 7.1259e-13, 5.3795e-13, 5.4967e-10, 1.7121e-11, 9.7206e-12, 4.1259e-12,\n 1.5431e-12, 7.7767e-10, 6.7803e-10, 1.7852e-11, 3.4672e-12, 1.5528e-10,\n 1.0128e-11, 1.0339e-12, 3.8043e-11, 1.4116e-12, 2.9115e-12, 1.7447e-13,\n 6.0280e-14, 1.3285e-10, 4.0311e-11, 6.6985e-10, 2.1139e-13, 1.7460e-11,\n 9.2977e-12, 4.7002e-11, 6.3521e-12, 2.7076e-11, 3.2318e-10, 1.8957e-10,\n 6.3724e-13, 4.3130e-13, 3.8642e-12, 2.4666e-10, 3.2956e-09, 8.8108e-11,\n 3.0150e-11, 6.0121e-11, 2.0526e-12, 1.5887e-11, 3.4660e-12, 2.3365e-11,\n 1.8607e-10, 2.0223e-13, 3.5692e-10, 4.1857e-10, 3.1410e-12, 1.1226e-12,\n 8.8388e-10, 1.0576e-12, 8.4204e-12, 2.9262e-10, 9.7967e-14, 1.5346e-12,\n 1.1024e-13, 2.4739e-13, 8.5303e-12, 1.2906e-10, 3.8799e-13, 1.8085e-11,\n 5.1481e-11, 8.4104e-11, 1.8908e-12, 3.1821e-14, 2.4012e-12, 1.3586e-12,\n 2.1049e-12, 2.3156e-11, 3.0358e-10, 2.2701e-09, 9.7673e-14, 3.7025e-13,\n 1.9736e-11, 1.0220e-09, 2.3165e-11, 1.2868e-12, 2.6565e-09, 2.6633e-13,\n 1.1228e-13, 3.2883e-12, 6.0289e-12, 1.1425e-12, 1.8174e-11, 8.7903e-12,\n 4.3728e-12, 1.1900e-11, 2.2878e-12, 1.5451e-09, 1.7469e-12, 2.3319e-09,\n 1.2786e-12, 2.8564e-12, 2.1316e-10, 3.4441e-13, 1.4715e-09, 4.6558e-10,\n 8.3863e-11, 4.6879e-13, 2.9110e-12, 1.5810e-11], device='cuda:0')" + }, + "41": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 3.0628e-16, -4.2582e-14, 1.3874e-14, 6.3964e-15, -9.5274e-15,\n -1.6756e-14, -1.0143e-15, 1.5861e-16, 2.6927e-15, -9.5018e-15,\n 1.0516e-15, 4.2511e-17, -5.1207e-15, -7.1115e-15, -1.1328e-14,\n -1.4289e-14, -1.0210e-14, 2.0314e-16, 5.9364e-15, -1.2176e-15,\n 3.0560e-15, -7.1256e-15, -2.2737e-15, 5.5271e-15, -4.7853e-15,\n -3.6075e-15, 1.0613e-15, 7.6981e-16, -3.1924e-15, 4.5692e-16,\n 1.3883e-14, 1.4202e-15, -6.4837e-16, -1.3285e-14, -7.0530e-16,\n -2.2556e-14, -1.5300e-14, 3.6322e-15, -1.8383e-14, -9.6519e-16,\n -7.1327e-16, -8.4773e-16, -7.0229e-15, 1.8378e-15, -2.5423e-14,\n -8.2388e-15, -9.0034e-15, -4.1833e-15, -4.3934e-15, 3.3844e-15,\n -2.2194e-15, 1.0945e-15, 6.1025e-15, 3.9028e-15, 3.9031e-16,\n -4.9899e-15, -1.0214e-14, -1.0000e-15, -8.7003e-15, -4.1071e-15,\n -1.1834e-14, -1.7122e-14, -7.0066e-15, -4.1698e-15, 1.8411e-15,\n -1.0626e-14, 1.1531e-15, 1.5605e-15, -3.6599e-15, 5.2806e-15,\n -9.8463e-16, 1.7537e-14, 2.2501e-16, 2.3815e-15, 1.8073e-15,\n -4.3071e-15, 1.9643e-15, 6.4244e-16, -2.4708e-14, -7.5929e-16,\n 2.5290e-15, -9.8338e-15, 7.3937e-15, -8.2308e-16, -1.7189e-15,\n -7.3433e-15, -5.6942e-15, -2.2000e-14, -2.5566e-15, -1.6842e-15,\n -3.1297e-16, 1.0368e-16, 2.6372e-15, 6.9302e-15, -2.6544e-14,\n -4.5410e-16, -2.3856e-14, -2.0350e-14, 4.3770e-15, 3.2490e-15,\n -7.8474e-15, -1.5753e-15, -2.7742e-15, -8.0617e-15, -6.2069e-15,\n -7.0114e-15, -1.6846e-16, 2.3153e-14, 1.1090e-16, -9.6916e-15,\n 2.8781e-15, -1.0518e-14, -7.4350e-15, 7.8225e-17, 4.0782e-15,\n -1.5294e-15, 5.8401e-16, 2.1488e-15, 6.7314e-16, 8.3741e-16,\n 2.4417e-15, 1.0158e-15, 3.1639e-16, -6.3565e-15, -1.7218e-15,\n -8.9949e-15, -7.1974e-15, -2.2636e-15, -6.5447e-15, -2.1282e-14,\n 7.5652e-16, -6.2127e-15, -1.7435e-16, -1.6641e-14, -8.2764e-15,\n -1.3356e-14, -4.6222e-15, 8.7811e-15, -7.3375e-16, -4.3357e-16,\n -8.2075e-15, -4.4183e-14, -1.9124e-16, 8.1180e-16, 7.6652e-17,\n 1.1077e-15, 3.1663e-15, -1.4522e-14, 2.2614e-15, 2.6719e-16,\n -3.5209e-16, 1.4963e-15, -1.2680e-16, -8.5086e-15, -7.9656e-16,\n -6.7868e-15, 1.2819e-15, -2.4987e-15, -1.6334e-15, 5.1828e-15,\n -5.2035e-15, -4.9301e-15, -5.0711e-15, -4.5527e-15, -7.1427e-15,\n -6.5459e-16, -6.0067e-16, -1.5714e-14, 4.4200e-15, 3.2649e-15,\n 1.1621e-14, 7.2334e-16, -9.2387e-15, -2.5016e-15, -6.8115e-15,\n 1.0847e-15, 8.2321e-16, -1.7074e-14, -6.2830e-15, -8.8714e-15,\n -1.3123e-14, 3.8566e-15, -2.2595e-14, -6.4702e-16, -1.5822e-14,\n -1.4154e-15, -1.2236e-14, 1.3355e-15, 2.6843e-16, -1.6359e-15,\n -2.0087e-15, -1.6049e-14, -2.2746e-14, 2.3761e-15, 8.6466e-15,\n -1.1261e-14, 6.3409e-16, -1.6813e-15, -1.7385e-15, 1.6387e-17,\n 3.1629e-15, -3.4821e-15, -3.3125e-16, -1.2753e-14, 1.1533e-18,\n 5.5900e-15, -5.2076e-15, 2.4791e-15, 2.2367e-15, -1.2985e-14,\n -8.8024e-15, -4.0029e-15, -1.6995e-14, -2.6085e-15, -1.0758e-15,\n -9.3209e-16, -1.0965e-14, 7.8827e-17, -4.6892e-16, -3.4226e-15,\n -1.2168e-14, -3.9247e-15, -1.3726e-14, -5.1761e-16, -1.0646e-15,\n 9.7310e-16, 6.8629e-15, -1.1933e-14, -1.3206e-14, 1.1180e-15,\n -2.0220e-14, -1.1999e-15, -1.2745e-15, -1.2724e-14, 6.1864e-16,\n 9.8884e-16, -1.9690e-15, -1.5340e-17, -1.9964e-14, 1.4365e-15,\n -3.4998e-15, 4.5875e-19, -1.5736e-14, -1.6650e-14, 8.5771e-15,\n 3.6314e-15, 7.0155e-15, 2.7407e-15, -4.9054e-15, 5.1684e-16,\n -1.2625e-14, -9.8151e-16, 7.5385e-16, -8.6106e-15, 9.8790e-16,\n -1.1373e-14, -9.0684e-15, 6.1580e-15, -7.3569e-15, 1.0327e-15,\n -1.1418e-15, -6.3472e-15, 1.1629e-15, -1.1429e-14, -3.0809e-15,\n 9.3067e-16, 1.9207e-15, -2.3371e-14, 4.5988e-16, -2.0592e-14,\n -1.8319e-15, 8.8466e-16, -2.4030e-15, -1.9169e-14, 1.2992e-17,\n 1.7093e-15, -1.9745e-14, -7.6297e-15, 7.0441e-15, -1.7262e-14,\n 2.1097e-15, 1.2235e-14, 1.4847e-15, -1.1587e-14, 1.1178e-15,\n 7.0702e-15, -7.1661e-15, -7.7205e-15, 2.1687e-15, 2.2691e-17,\n 3.6610e-15, 4.9738e-17, -3.1899e-14, 1.1775e-14, -1.7907e-15,\n -9.8253e-15, 2.6624e-15, -5.9485e-15, -6.8602e-15, -5.3083e-15,\n 2.7391e-15, -5.8719e-15, -5.3157e-15, -6.5015e-15, -2.3066e-14,\n -4.1118e-15, -1.4960e-15, -6.7830e-15, -1.3296e-14, -2.1927e-14,\n -2.7766e-15, -4.7055e-15, -1.7064e-14, 5.8128e-15, -2.8574e-15,\n -1.7961e-14, 1.0728e-15, 5.3547e-15, -1.3607e-14, 3.9291e-15,\n -3.5164e-15, 2.8399e-15, -2.1376e-16, 1.1068e-15, 1.8722e-15,\n 2.2341e-15, 2.2978e-16, -1.7220e-15, -1.1322e-15, -1.6328e-15,\n -9.0711e-15, -3.5156e-15, -2.5376e-17, -7.2765e-16, -6.3602e-15,\n 1.8644e-15, -8.4128e-16, -5.2382e-15, -2.8214e-15, -1.9094e-14,\n -2.9500e-15, 9.8047e-15, -1.2649e-14, -1.5226e-16, -4.6417e-14,\n -1.8528e-14, -6.3270e-16, 8.0826e-17, -1.7003e-14, -1.6366e-15,\n 4.2686e-15, -1.0112e-15, 1.5748e-15, -9.6347e-15, -1.3180e-15,\n -1.2474e-14, -1.7727e-14, -1.0550e-15, -1.5350e-14, -3.1001e-15,\n 1.2466e-15, -1.1323e-14, -9.3781e-16, -3.5111e-14, -5.2993e-15,\n 5.7376e-16, -1.0254e-14, -8.6780e-15, -5.4081e-15, -2.0844e-14,\n -1.7277e-16, -6.9194e-16, -1.2983e-14, -4.0087e-15, 1.3859e-16,\n -7.6328e-16, -1.1827e-15, -5.3243e-15, -4.1992e-15, -2.2724e-15,\n -6.6328e-15, 3.3234e-16, -1.9964e-16, -1.8420e-15, -6.4011e-15,\n -9.5939e-16, -1.5222e-14, 1.0610e-15, -4.3547e-15, -5.4532e-15,\n -7.4030e-16, 1.4741e-15, -1.7873e-14, -1.4791e-15, -3.0675e-14,\n 7.9269e-16, -2.4936e-14, -4.1124e-14, 3.2757e-15, -7.4607e-15,\n -1.7318e-14, -4.5753e-15, 5.7341e-16, -8.4556e-15, -1.4730e-14,\n -1.7508e-14, 5.0368e-15, -1.5683e-14, 2.8333e-15, -2.2215e-15,\n 4.6803e-15, -1.1363e-15, -4.1257e-14, 2.2562e-15, -6.2592e-15,\n -6.9720e-15, -5.2697e-16, -1.7277e-15, -1.4836e-14, -1.8094e-14,\n 1.1812e-16, -3.2336e-15, -1.1389e-14, -9.2548e-15, -1.1881e-14,\n 1.1342e-14, -1.2881e-14, -1.8852e-15, -1.8847e-15, 1.0573e-15,\n -1.0170e-14, -4.9075e-16, -5.7413e-16, 2.3694e-15, -3.5185e-15,\n -5.8463e-15, -3.3229e-15, -6.7119e-15, -2.5013e-15, 1.6714e-15,\n -4.6484e-16, 5.6542e-16, -8.9259e-15, 8.9312e-16, -1.3740e-15,\n -9.2611e-15, -2.8011e-14, -3.5828e-15, -5.0003e-15, -1.1044e-14,\n -1.4775e-14, -4.3255e-15, -3.2779e-16, -4.2883e-16, 1.8558e-14,\n -1.1907e-14, -6.6677e-15, -1.4541e-14, -8.0326e-16, -1.1513e-14,\n 1.7530e-15, -2.4372e-15, -2.7639e-14, -3.9358e-15, -9.6494e-15,\n -1.2679e-16, -6.0031e-16, -2.9189e-14, -1.4702e-14, 9.6270e-16,\n -9.8993e-16, 3.2518e-15, -8.0104e-15, -6.0576e-16, -5.1433e-16,\n -2.6149e-15, 1.9902e-15, -5.4637e-16, 4.2208e-16, -1.6081e-15,\n -1.9543e-16, -5.7229e-17, -9.1673e-15, -8.8581e-15, 1.6966e-15,\n -1.4714e-14, 2.3198e-15, -3.8153e-15, -1.5774e-16, -1.0312e-14,\n 3.2211e-16, -2.7582e-16, -8.7293e-16, 5.0035e-16, 1.4267e-15,\n -1.5012e-14, 1.7307e-16, -9.8235e-16, -5.0852e-15, 3.2934e-15,\n -9.6574e-15, -2.1937e-15, 1.8800e-15, -5.3291e-16, 6.8859e-16,\n -5.6228e-15, -1.2488e-15, -2.8493e-16, 1.4080e-14, -8.3609e-15,\n 1.5125e-15, -3.7265e-16, -2.5156e-15, 7.7660e-15, 1.3224e-15,\n -6.6124e-15, -2.2331e-15, -7.6314e-15, -1.3228e-14, -5.7132e-16,\n -1.2361e-14, 4.0819e-15, -2.1881e-15, 5.2224e-15, -6.8903e-15,\n -1.3493e-14, -2.6557e-15, -5.9654e-15, -2.8385e-15, -9.9695e-15,\n 1.5639e-15, 2.7775e-15, 1.1583e-15, -2.2272e-16, -1.7916e-14,\n -8.5339e-15, -2.3008e-16, 4.3631e-15, -9.6791e-15, -1.9998e-14,\n 5.7812e-17, -2.6193e-14, -6.8351e-15, -9.4524e-15, -1.8719e-16,\n -3.5434e-14, -8.6099e-16, 8.3178e-15, -2.6790e-14, 1.1819e-14,\n -1.9749e-15, -2.2959e-16, 1.2022e-15, -8.6680e-15, 2.4329e-15,\n -7.1859e-16, 8.8772e-16, -3.0722e-15, -8.7384e-16, -2.4298e-14,\n -5.9222e-15, -1.5237e-15, -2.1230e-16, 2.4383e-15, -2.1963e-15,\n 4.0028e-15, -1.0452e-14, 3.4091e-15, 3.1948e-15, -4.5977e-15,\n -1.6642e-15, 1.1909e-15, -2.6454e-16, -1.9018e-14, -1.7495e-14,\n -3.2164e-14, -2.0628e-15, -1.3316e-16, -1.2998e-15, 4.5008e-16,\n -3.3920e-14, -1.6263e-14, -1.3873e-14, -5.5140e-15, -7.3584e-15,\n -1.6054e-14, -1.6393e-14, -3.5165e-15, 6.2304e-16, -3.7871e-15,\n 4.2071e-16, -3.5585e-15, -4.5763e-15, 1.1522e-15, -6.8404e-18,\n -7.5197e-15, 2.9965e-15, -1.3536e-14, -1.5613e-14, 1.1447e-15,\n -1.2754e-15, -8.8798e-15, -1.8457e-14, 4.0434e-15, -8.2027e-16,\n -3.0614e-17, -7.3599e-15, 2.3653e-15, 4.6266e-15, -2.6440e-14,\n 5.3907e-15, -2.0685e-15, -3.5014e-15, -2.8400e-15, 2.4918e-15,\n -7.2488e-15, -1.3238e-14, -7.6980e-15, -5.1786e-15, 2.6530e-15,\n -6.0376e-15, -1.6564e-14, -3.3970e-14, -2.4503e-14, -2.7066e-16,\n 9.8476e-16, -1.2130e-15, -1.4424e-14, -3.1007e-16, -5.1724e-15,\n -4.1320e-16, 2.3866e-15, -9.3546e-15, -4.1752e-15, -1.0632e-14,\n -6.3774e-15, -1.4091e-14, 1.0699e-15, 3.7236e-17, -3.8249e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.9623e-12, 2.5415e-11, 1.6371e-11, 1.3747e-10, 5.2656e-11, 7.9950e-11,\n 5.3892e-12, 7.6078e-12, 1.7260e-11, 2.6577e-11, 1.4363e-12, 5.6568e-12,\n 8.9668e-10, 1.5900e-10, 1.0030e-10, 3.3116e-11, 9.3004e-11, 6.6399e-12,\n 1.2887e-10, 3.2019e-12, 9.7197e-11, 2.2426e-12, 1.5967e-11, 1.3795e-12,\n 1.8875e-11, 6.6598e-10, 8.1259e-11, 3.0693e-10, 2.5431e-11, 2.4263e-12,\n 5.1237e-10, 4.1285e-10, 2.1115e-12, 6.3186e-10, 2.6695e-10, 1.0197e-10,\n 2.2864e-10, 3.2401e-11, 3.4722e-10, 1.9325e-13, 2.0596e-12, 6.6838e-11,\n 2.5612e-10, 3.5911e-13, 9.2707e-12, 4.2314e-10, 5.6005e-10, 1.0249e-11,\n 4.3762e-12, 3.4733e-12, 1.2331e-11, 2.0928e-10, 3.3125e-11, 1.1836e-11,\n 1.8652e-10, 1.6658e-11, 1.8665e-10, 1.3240e-12, 1.3981e-10, 1.0253e-11,\n 1.1857e-11, 1.5211e-10, 2.0066e-12, 1.3399e-10, 1.9666e-12, 2.7001e-11,\n 2.5157e-12, 7.1197e-11, 3.2911e-10, 5.8258e-10, 6.9610e-10, 1.2224e-10,\n 2.5309e-12, 5.2221e-11, 7.4927e-11, 3.8070e-10, 4.1849e-12, 3.4504e-11,\n 9.8167e-11, 1.2579e-09, 3.7455e-11, 2.6631e-11, 7.7904e-12, 1.6336e-10,\n 5.9811e-11, 6.4799e-13, 9.4063e-13, 6.1818e-12, 5.5788e-11, 4.8310e-10,\n 4.9589e-11, 6.1735e-11, 3.0601e-12, 5.1382e-10, 4.8438e-12, 7.3294e-10,\n 4.4904e-11, 6.6563e-10, 1.8167e-10, 6.8493e-12, 5.2940e-12, 6.0985e-10,\n 7.3519e-13, 8.2185e-12, 1.5699e-10, 7.5697e-11, 1.7929e-13, 3.6883e-11,\n 1.2934e-11, 1.9804e-12, 6.2620e-12, 6.6731e-10, 1.8983e-11, 5.9257e-13,\n 1.5649e-11, 2.7282e-13, 2.3915e-12, 3.0687e-10, 1.7163e-13, 4.1393e-11,\n 4.9622e-10, 5.9204e-10, 6.4401e-11, 1.6950e-10, 1.2543e-11, 1.1454e-12,\n 2.6385e-11, 8.7113e-12, 4.1987e-12, 1.5482e-11, 6.9275e-13, 2.3457e-12,\n 9.4915e-12, 2.6920e-10, 7.7658e-11, 4.9396e-10, 1.1804e-12, 1.8599e-10,\n 1.3969e-10, 4.2379e-12, 3.0164e-11, 7.2250e-11, 7.0777e-12, 4.1753e-12,\n 5.1731e-13, 2.3110e-11, 3.1306e-12, 1.4067e-09, 2.2257e-10, 3.4178e-12,\n 7.7881e-12, 6.2303e-12, 4.8178e-12, 1.5589e-10, 6.8205e-13, 1.6451e-10,\n 5.6121e-13, 7.9424e-11, 4.9511e-13, 2.0517e-11, 2.3919e-10, 5.6176e-12,\n 1.0031e-10, 4.3433e-13, 2.7205e-12, 1.3082e-13, 1.1977e-10, 7.2698e-10,\n 3.4726e-11, 4.3601e-12, 2.1918e-12, 1.1721e-11, 8.6193e-13, 4.1261e-11,\n 6.6266e-10, 4.3106e-12, 7.5349e-13, 6.6290e-10, 1.0527e-12, 3.1243e-11,\n 1.0552e-11, 1.0094e-10, 9.2635e-12, 1.9392e-10, 1.0018e-10, 6.1048e-12,\n 2.7126e-10, 2.7121e-10, 2.0754e-12, 6.5679e-11, 3.0342e-10, 1.7443e-11,\n 2.7485e-10, 1.6441e-11, 4.7543e-10, 8.6321e-11, 4.1411e-13, 4.2693e-11,\n 9.1260e-10, 4.2444e-11, 2.7199e-12, 1.9660e-10, 6.5732e-12, 5.3756e-11,\n 6.8348e-13, 1.6020e-10, 5.7897e-10, 1.5331e-12, 6.0687e-11, 1.2737e-10,\n 4.2699e-10, 4.8172e-11, 2.9107e-11, 3.8081e-10, 1.2630e-10, 8.4598e-12,\n 2.0207e-11, 3.3192e-13, 1.2641e-10, 2.2297e-11, 6.5498e-11, 1.3222e-11,\n 4.1475e-10, 1.0538e-10, 1.1717e-12, 2.9312e-13, 7.2983e-10, 5.0946e-10,\n 2.3534e-10, 3.1009e-13, 7.4566e-11, 2.3161e-11, 2.2002e-11, 2.5275e-11,\n 6.0474e-11, 1.0961e-09, 9.8258e-12, 1.9400e-12, 2.4579e-11, 1.1595e-11,\n 2.4168e-11, 1.0334e-12, 1.6178e-10, 1.6560e-11, 7.1056e-12, 7.0259e-12,\n 4.7863e-12, 3.9240e-10, 1.5122e-11, 1.3711e-11, 1.2005e-10, 4.2556e-12,\n 5.5277e-11, 5.2028e-10, 2.3573e-10, 6.8381e-10, 4.5783e-13, 4.1182e-11,\n 1.3578e-11, 2.9541e-12, 2.8957e-12, 1.4590e-11, 9.3687e-12, 8.5103e-11,\n 2.8944e-10, 1.7431e-11, 8.8589e-12, 2.6417e-11, 1.0483e-11, 4.9230e-10,\n 8.8705e-11, 7.2451e-13, 2.1165e-10, 8.7060e-11, 4.8727e-12, 4.0621e-12,\n 1.1722e-11, 1.0214e-11, 1.0405e-10, 2.7332e-10, 1.0187e-09, 1.5990e-10,\n 6.8443e-11, 5.6896e-10, 2.1418e-11, 1.6768e-12, 7.0152e-11, 3.4078e-10,\n 8.0395e-10, 1.2900e-12, 2.8842e-11, 5.6940e-12, 6.5422e-10, 2.6356e-12,\n 5.4489e-11, 3.5021e-10, 1.1282e-11, 2.3703e-11, 7.7604e-11, 2.1775e-11,\n 3.1155e-12, 7.4699e-13, 9.3485e-10, 2.5453e-10, 2.5875e-11, 1.1659e-12,\n 1.5080e-11, 2.9560e-10, 9.6596e-13, 1.3363e-12, 5.9667e-12, 4.7016e-10,\n 6.4546e-11, 1.0613e-09, 6.5534e-10, 5.6753e-13, 5.4518e-14, 4.0066e-10,\n 3.5732e-10, 6.7179e-10, 7.1866e-13, 1.9179e-12, 4.1654e-11, 5.0857e-11,\n 2.5804e-11, 7.8842e-13, 7.6476e-11, 1.2141e-11, 1.8382e-12, 8.2440e-10,\n 2.2758e-10, 2.4829e-11, 4.3582e-10, 6.4254e-11, 5.0858e-10, 1.5124e-11,\n 7.8361e-12, 7.2539e-12, 1.2695e-10, 1.0377e-11, 3.3563e-10, 1.4695e-11,\n 1.4935e-11, 1.1421e-10, 5.6038e-10, 1.4977e-11, 2.9606e-12, 6.7667e-12,\n 1.7264e-10, 3.0658e-11, 1.5696e-12, 6.1831e-11, 3.3051e-12, 4.9687e-11,\n 3.1387e-11, 1.8549e-12, 9.2759e-10, 2.3092e-12, 2.8782e-12, 1.5201e-10,\n 8.4813e-11, 2.5792e-12, 1.1816e-11, 3.6740e-10, 2.3824e-11, 1.7274e-10,\n 5.7787e-12, 3.4913e-10, 9.8582e-10, 4.7975e-10, 1.1087e-10, 1.9827e-12,\n 3.4148e-11, 1.6027e-11, 8.6625e-13, 3.6480e-11, 1.0690e-11, 4.4124e-10,\n 4.3210e-10, 8.0875e-10, 1.5242e-11, 9.3373e-13, 1.8821e-12, 5.2207e-11,\n 7.9282e-14, 3.1896e-11, 1.2440e-11, 8.6946e-12, 1.9444e-10, 1.2498e-10,\n 1.4722e-12, 2.5450e-11, 4.2810e-10, 6.5279e-11, 3.0570e-10, 3.0815e-11,\n 4.9025e-10, 3.3718e-11, 2.0597e-11, 1.2630e-09, 5.9971e-12, 2.5414e-11,\n 4.1878e-12, 6.7071e-11, 1.0762e-11, 2.1687e-10, 9.1888e-11, 8.9103e-11,\n 2.0561e-09, 9.2256e-13, 1.1106e-11, 4.4869e-11, 1.5126e-10, 2.2089e-11,\n 1.7835e-11, 2.8623e-11, 2.0428e-11, 7.5581e-13, 8.1136e-11, 2.0322e-12,\n 2.5426e-13, 1.3358e-10, 7.1573e-10, 2.7173e-11, 1.0140e-09, 3.7980e-10,\n 2.7118e-10, 7.9086e-10, 1.0421e-12, 3.5988e-11, 2.2066e-11, 8.1036e-12,\n 1.1508e-10, 1.1594e-10, 4.1426e-11, 5.4811e-12, 3.3429e-10, 1.6259e-11,\n 2.4923e-10, 2.6346e-11, 9.9514e-11, 2.8216e-12, 1.8754e-11, 7.3396e-10,\n 1.1484e-10, 3.9210e-10, 9.7583e-10, 1.5179e-10, 9.3320e-11, 4.6879e-11,\n 5.3591e-11, 7.4172e-11, 2.1982e-13, 3.9527e-11, 1.2607e-12, 6.6861e-12,\n 4.0816e-10, 5.5349e-11, 8.7399e-11, 4.3473e-12, 9.4983e-10, 1.8390e-10,\n 3.8335e-10, 8.0306e-11, 2.4761e-12, 1.0819e-10, 9.6640e-11, 1.0604e-11,\n 9.3353e-12, 5.0784e-11, 1.3218e-12, 3.5277e-10, 1.8546e-11, 1.6237e-12,\n 8.9424e-12, 1.1711e-10, 3.8218e-12, 6.7441e-12, 1.6830e-13, 6.6564e-11,\n 1.0278e-12, 1.3874e-12, 2.4830e-12, 3.5935e-11, 5.7888e-13, 9.1697e-10,\n 2.0620e-10, 4.3936e-10, 4.8773e-13, 1.9807e-11, 7.7623e-12, 6.8453e-10,\n 7.1957e-12, 1.9063e-12, 1.2532e-12, 1.2596e-11, 8.3198e-13, 1.0305e-10,\n 1.3628e-12, 2.6672e-11, 1.1843e-10, 8.2777e-10, 1.1898e-11, 3.1614e-12,\n 9.7245e-13, 2.4636e-12, 3.1731e-11, 7.6696e-12, 1.5580e-11, 2.3128e-10,\n 9.4852e-13, 2.4805e-12, 9.2372e-12, 2.5352e-10, 2.2592e-11, 7.3730e-11,\n 6.2648e-13, 9.4320e-11, 2.2692e-10, 1.5379e-12, 1.4326e-09, 1.0215e-10,\n 2.9765e-11, 2.0080e-11, 2.8497e-12, 7.7299e-12, 7.8169e-12, 8.8550e-10,\n 2.9206e-11, 1.0586e-11, 1.2131e-12, 2.0633e-11, 4.7656e-10, 5.8134e-12,\n 5.1121e-10, 3.5865e-11, 2.7150e-10, 1.9733e-10, 1.7095e-11, 5.8127e-11,\n 2.6348e-13, 9.1960e-12, 5.9492e-10, 2.4183e-11, 3.9102e-12, 1.0841e-11,\n 8.8917e-13, 6.2786e-10, 3.7805e-10, 2.8642e-11, 6.3336e-12, 2.2529e-10,\n 4.3384e-12, 3.7904e-11, 1.4678e-11, 4.3259e-11, 1.2869e-12, 6.8241e-13,\n 6.1680e-13, 1.2316e-10, 1.0463e-10, 3.5542e-10, 4.3891e-13, 1.7295e-10,\n 2.9635e-10, 8.3074e-11, 8.3982e-11, 1.2081e-11, 9.6983e-11, 5.4681e-10,\n 4.2789e-12, 2.4760e-13, 1.3100e-12, 5.2924e-10, 2.3482e-09, 8.6900e-11,\n 4.3423e-11, 2.6988e-11, 2.9523e-12, 1.8764e-11, 5.6102e-12, 2.8289e-11,\n 6.3540e-10, 9.0556e-13, 2.4508e-10, 4.9089e-10, 2.1861e-11, 1.2447e-11,\n 3.3177e-10, 4.3991e-11, 4.0487e-12, 2.7264e-10, 3.2641e-13, 6.7928e-13,\n 2.6468e-12, 4.8941e-12, 3.6336e-11, 4.5855e-11, 3.8480e-12, 5.8875e-11,\n 2.5376e-10, 1.4683e-10, 9.8814e-12, 4.3416e-13, 6.5184e-11, 4.2795e-13,\n 5.7021e-11, 1.9467e-10, 2.4419e-10, 8.7582e-10, 7.6077e-13, 1.8542e-13,\n 8.2354e-11, 5.6929e-10, 9.1814e-12, 2.1057e-12, 6.2714e-10, 3.3999e-12,\n 6.1690e-13, 2.1957e-12, 2.2077e-11, 1.7093e-11, 1.7254e-11, 5.4236e-12,\n 1.1227e-10, 5.9480e-12, 1.3511e-11, 5.7334e-10, 1.7686e-11, 1.2296e-09,\n 3.0359e-11, 8.4170e-11, 1.3392e-10, 8.0467e-12, 6.5087e-10, 1.8487e-10,\n 2.0543e-10, 4.9938e-13, 1.2911e-12, 7.6046e-12], device='cuda:0')" + }, + "42": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 4.9593e-18, 1.6031e-17, -5.6052e-45, ..., 2.8503e-16,\n 1.5337e-15, 5.6052e-45],\n [-2.9402e-16, -2.1899e-16, -5.6052e-45, ..., -5.3590e-15,\n -4.2761e-14, -5.6052e-45],\n [ 5.8902e-17, 1.1544e-17, 5.6052e-45, ..., 1.0087e-15,\n 9.8062e-15, 5.6052e-45],\n ...,\n [ 4.7285e-17, 2.5846e-18, 5.6052e-45, ..., -2.6654e-15,\n -2.7228e-15, 5.6052e-45],\n [ 2.9725e-17, 2.0674e-17, 5.6052e-45, ..., 1.2702e-15,\n 7.3385e-15, 5.6052e-45],\n [-3.2751e-17, -2.7061e-17, 5.6052e-45, ..., 1.7092e-15,\n -1.2412e-15, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.3768e-11, 2.5328e-11, 2.4568e-15, ..., 2.7446e-11, 8.1067e-12,\n 1.6784e-13],\n [6.7735e-12, 7.5433e-12, 2.0650e-16, ..., 1.3843e-11, 3.0560e-12,\n 2.7751e-14],\n [1.1601e-12, 5.8933e-13, 6.2404e-17, ..., 9.6840e-13, 2.9514e-13,\n 6.4220e-14],\n ...,\n [3.1067e-12, 4.2975e-12, 2.8057e-17, ..., 3.2372e-12, 4.6068e-12,\n 8.8441e-15],\n [1.5717e-12, 1.8440e-12, 2.6754e-16, ..., 2.1272e-12, 4.8408e-13,\n 5.9892e-15],\n [1.1331e-10, 9.0511e-11, 1.8778e-15, ..., 1.2624e-10, 2.6545e-11,\n 1.3412e-13]], device='cuda:0')" + }, + "43": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 1.4084e-13, -4.6603e-12, 8.5508e-13, 6.0626e-13, -4.8540e-13,\n -3.8778e-14, -1.4178e-13, 2.1760e-13, 2.5287e-13, -2.6906e-13,\n -3.4711e-13, 3.8799e-13, 1.5226e-13, 2.8806e-13, -3.9646e-13,\n 3.1545e-13, -1.3907e-12, 9.5754e-13, 6.9810e-13, -1.1850e-12,\n 3.6191e-13, -1.6155e-12, 7.3794e-13, -3.2573e-13, -2.1960e-13,\n -4.2691e-13, -1.2818e-13, 1.1223e-12, -1.1801e-12, -4.8430e-14,\n 1.1998e-12, 1.6276e-12, 3.8406e-13, -5.0475e-13, 6.3698e-13,\n 2.3698e-13, -5.2576e-13, 4.1815e-13, -2.4256e-12, 1.4271e-12,\n 4.5971e-13, 2.9602e-13, -5.8051e-13, 5.5682e-14, -6.7155e-13,\n -5.0895e-15, -5.1105e-14, 3.2246e-13, -1.0272e-13, -2.5704e-13,\n 1.6297e-12, 7.3441e-13, 3.4108e-13, 2.7027e-13, 3.3189e-13,\n 2.5178e-13, -1.1886e-12, -7.8114e-13, 2.0933e-14, -8.1501e-15,\n -1.1393e-12, -4.6087e-13, 5.4492e-14, 5.6766e-14, -2.0452e-12,\n 9.8019e-13, -1.3046e-12, 1.1086e-12, -2.4047e-13, 4.4237e-13,\n -1.5076e-13, -1.8693e-13, -1.7149e-12, -1.4771e-14, -9.9602e-14,\n 1.8059e-13, -1.1424e-12, 1.1453e-12, -3.7547e-12, 5.6409e-13,\n 5.5779e-13, -4.4194e-13, -4.0618e-13, 1.4218e-13, -1.0386e-12,\n 3.4658e-13, -2.6519e-13, -2.8335e-12, -3.6839e-13, 1.0380e-13,\n 8.3710e-13, 1.1885e-12, -1.9981e-13, -5.6172e-13, -1.8847e-12,\n -1.4382e-12, 6.5644e-14, 3.0216e-13, 4.7135e-13, 2.9294e-13,\n -9.2363e-13, 3.5451e-14, -2.5127e-13, 5.2168e-13, -9.5836e-14,\n -5.2619e-13, -5.0495e-14, 2.9331e-13, 7.2147e-13, -8.8941e-13,\n -2.6252e-12, 6.7889e-14, 2.3748e-13, 2.7088e-14, 9.6656e-14,\n 3.1316e-13, 3.5971e-13, 6.4689e-13, -2.5361e-13, -1.9586e-12,\n -3.1169e-13, -4.2820e-13, 5.3684e-13, -2.2195e-14, -1.1062e-12,\n -4.6048e-13, -3.7480e-12, -6.6414e-13, -3.5749e-14, 2.1138e-13,\n 3.7384e-13, 7.7688e-14, 1.0124e-12, -1.0262e-13, 2.9291e-13,\n 4.8682e-13, -6.0834e-13, 1.2445e-13, 8.1283e-14, -1.4756e-13,\n -8.8932e-14, -2.9356e-14, 7.6700e-13, 1.9497e-12, 3.7712e-13,\n 4.1316e-13, 9.7733e-14, 2.6540e-13, 2.1199e-13, 1.9115e-12,\n 4.9822e-13, -5.0726e-13, 6.0975e-13, -8.7232e-14, 8.6448e-13,\n 7.7366e-13, 1.1420e-12, -3.0801e-14, 4.9327e-13, 2.0980e-12,\n 5.8397e-13, 1.7894e-13, -1.6508e-12, 3.2037e-13, 3.0170e-13,\n 6.3167e-13, -9.7252e-14, -2.5293e-14, 3.5605e-13, 1.0679e-12,\n 8.0193e-13, -9.1174e-14, -8.3714e-13, -5.7587e-13, -1.0256e-12,\n 5.2446e-13, 3.8112e-13, 8.7248e-13, 8.3875e-14, 2.4372e-13,\n 4.5907e-13, 2.6449e-13, 8.4874e-14, 5.8520e-13, -6.6151e-13,\n 9.0612e-13, 4.9812e-13, 3.4739e-14, 1.8352e-13, 1.0812e-12,\n -9.4935e-14, 2.7227e-13, 4.9235e-13, 1.0286e-12, 1.2355e-12,\n -9.6684e-13, -6.9557e-13, 1.1231e-12, 1.2616e-12, 5.3540e-13,\n -5.4230e-13, -4.2045e-13, -6.5173e-14, -4.6315e-13, 5.7391e-13,\n 2.3261e-12, -2.7108e-13, -1.0755e-12, 4.0325e-13, 2.4584e-13,\n -1.1184e-14, 3.7553e-13, 8.5299e-14, 1.6056e-13, 7.4454e-13,\n 3.6506e-13, -1.1532e-12, 2.7617e-13, 3.0846e-13, 2.7743e-13,\n 1.4930e-13, 9.4970e-14, 2.0429e-13, 2.4069e-13, 5.9424e-13,\n 3.6673e-13, 1.2434e-12, -1.2079e-12, -1.6790e-13, -3.7494e-14,\n -2.6617e-13, 3.5873e-13, 5.1263e-13, -1.4079e-12, -4.2357e-13,\n -4.7228e-13, 1.7439e-13, 2.4757e-12, -5.7867e-13, 9.8305e-13,\n -4.9635e-13, 1.8485e-13, 4.5683e-13, 3.6051e-14, 5.4229e-13,\n -6.9907e-13, 6.3465e-14, 1.1545e-12, -5.6622e-14, -6.9326e-13,\n 1.8229e-13, 9.2442e-13, 3.7080e-14, -9.8124e-13, 2.4691e-13,\n -2.7973e-13, -7.5290e-13, 1.3872e-13, 1.7601e-13, -1.6381e-14,\n -9.6390e-13, 4.9536e-13, -2.6486e-14, 1.5662e-13, 3.5619e-13,\n 1.8848e-12, 1.4666e-12, 2.0543e-13, 7.3839e-13, -3.3536e-12,\n 3.8160e-13, 3.7763e-13, 5.7338e-13, 1.1914e-13, 9.7066e-13,\n -1.3080e-12, -1.6838e-13, 7.1176e-13, 1.8401e-13, 5.0285e-13,\n -3.4538e-13, 8.3049e-13, -3.8950e-13, -7.6585e-13, 9.6063e-14,\n -4.1093e-12, 7.2163e-14, -1.4762e-12, -6.0567e-13, 1.8137e-12,\n 2.8729e-13, -9.9220e-13, -1.3012e-13, 1.1957e-12, -1.2800e-13,\n 4.6354e-13, -1.1925e-12, -1.1362e-12, 4.0426e-13, 5.0875e-13,\n 8.1587e-13, -4.8788e-14, -5.0080e-13, -6.1989e-14, 1.4640e-13,\n 2.0496e-12, 4.1470e-13, 3.6385e-13, -3.9397e-14, 4.0783e-13,\n 1.8234e-13, -2.1069e-14, -2.0711e-12, 2.6200e-13, 8.7771e-14,\n 3.4673e-14, -9.9545e-13, 1.1085e-13, -1.0178e-12, 8.9764e-13,\n 1.0305e-13, 7.5775e-13, 1.7118e-13, 1.1554e-12, 3.8318e-14,\n -7.6736e-14, 2.9620e-13, 8.1287e-13, 6.9146e-13, -1.2920e-13,\n -3.7142e-14, 2.7586e-13, 1.7228e-13, 1.4906e-12, -8.1321e-13,\n 4.1645e-13, 6.9489e-13, -5.0638e-13, -2.1799e-12, -6.3736e-13,\n -6.4776e-13, 3.5846e-13, -1.5667e-12, -2.5446e-13, -5.5394e-13,\n -2.3964e-12, 3.2928e-13, 1.5646e-12, -2.7302e-14, -3.8080e-14,\n 1.5695e-13, 2.1233e-13, 1.0104e-12, -3.0359e-13, 1.7964e-12,\n -1.6731e-13, -2.8330e-12, -2.0408e-13, 3.2663e-13, -1.6444e-13,\n 1.9564e-13, 2.4766e-13, 5.8990e-14, 1.6554e-14, 5.4672e-13,\n 3.2138e-13, -7.3811e-13, 8.6438e-13, -4.7435e-13, -1.4341e-12,\n 2.8559e-13, 7.4342e-13, -9.6929e-13, 2.6139e-13, -1.7454e-13,\n 6.5237e-13, 2.6066e-13, 4.6964e-13, -1.0992e-13, 2.7825e-14,\n -5.2849e-13, 5.6287e-13, 1.0115e-12, 3.2629e-13, 8.0958e-14,\n 9.5254e-14, -9.9858e-13, 1.0109e-12, -1.1379e-12, -1.3884e-13,\n 1.1758e-12, -7.6868e-15, 3.6892e-13, 2.2470e-12, -1.5765e-13,\n 6.5995e-14, -2.3618e-13, -1.7663e-12, -2.6482e-12, -3.4760e-14,\n 8.6910e-14, -2.9432e-14, -4.2915e-13, 9.7504e-13, 2.9011e-13,\n -5.6624e-13, 6.7949e-13, -5.4823e-13, -2.6412e-13, -3.9642e-14,\n -7.4424e-13, -4.3081e-14, -9.7084e-14, 2.2098e-13, -2.1483e-13,\n -6.6951e-13, 1.0516e-12, 1.5270e-12, -1.7138e-12, -3.9310e-13,\n 1.2724e-12, 7.3361e-14, -5.0778e-13, 1.4688e-13, 6.6612e-14,\n 1.3000e-13, -1.0626e-12, 2.2772e-13, 9.7123e-13, -5.1197e-14,\n 1.8015e-13, 1.0776e-12, -7.5132e-13, -4.0074e-15, -1.5363e-12,\n -7.4194e-13, -3.2611e-13, -1.2829e-14, 5.6101e-14, 2.0122e-13,\n -5.8488e-13, 1.7620e-13, -7.5453e-13, 2.2955e-13, 3.3598e-13,\n 2.8568e-13, -7.0418e-13, 2.1823e-13, -5.4460e-14, -9.1567e-13,\n 2.5737e-14, -4.0265e-13, 3.0836e-13, -8.5007e-13, 1.8159e-12,\n -3.5722e-13, 1.0706e-13, 2.5971e-13, 2.9767e-13, -1.2573e-13,\n 5.2244e-13, 1.7703e-13, 2.4917e-13, -3.0696e-12, -1.5114e-12,\n 1.0829e-12, 2.1707e-13, -6.0160e-13, -1.4273e-13, 4.0795e-13,\n 2.6366e-13, 1.9324e-13, 2.0302e-13, -3.3677e-13, 1.3791e-12,\n -5.4682e-13, 1.0886e-13, 2.7610e-13, 7.8275e-13, 1.9427e-13,\n 1.9610e-13, 7.0895e-13, 6.8605e-13, 1.4482e-13, 7.1120e-13,\n 6.5834e-14, 1.0804e-12, -1.7516e-12, 2.0101e-13, -2.5363e-12,\n 1.6603e-12, -5.8052e-13, 7.4544e-14, 2.8809e-13, 5.3559e-13,\n -1.7178e-12, 9.5264e-13, 4.8354e-13, 5.6188e-13, 4.7976e-13,\n 1.6190e-13, 5.7633e-13, 6.6205e-13, -1.6510e-13, 1.3203e-13,\n 2.1106e-13, -1.6901e-12, -5.1563e-13, 6.5840e-14, 2.6378e-13,\n 5.3315e-15, 8.1536e-13, 4.6049e-13, 9.1735e-14, 3.5015e-13,\n -5.6241e-15, -4.3591e-13, 1.7381e-13, 3.1702e-13, 1.5923e-13,\n -5.1150e-14, 6.2972e-13, 8.4810e-13, -6.2989e-14, 7.8450e-14,\n 1.1875e-13, 2.0832e-13, 1.0971e-13, 1.1553e-13, -7.0389e-13,\n 2.5668e-14, 9.5648e-13, -8.6743e-14, 5.1827e-13, 2.5672e-13,\n 1.1233e-12, 2.1508e-13, 1.7401e-12, 7.1632e-13, -1.5729e-12,\n -5.1280e-13, -1.4483e-12, 5.9610e-13, 3.4951e-15, -1.3689e-12,\n -9.3014e-13, -6.8744e-14, 1.4542e-12, 4.4600e-13, 1.1263e-12,\n -1.9804e-13, 1.5542e-13, 3.5514e-13, -7.7107e-13, 2.8580e-13,\n 1.0857e-12, 4.0645e-13, 2.9524e-13, 2.9965e-13, 2.1768e-13,\n 1.3263e-12, 1.5368e-14, -2.3368e-13, 5.2248e-14, 4.7048e-13,\n -6.2845e-13, -3.2843e-13, -1.1135e-12, -5.0870e-15, -1.3372e-12,\n 3.5350e-14, 2.4185e-13, 6.5501e-13, -3.5090e-13, -3.6955e-12,\n 5.0926e-13, -4.8701e-13, 8.8792e-13, -1.3689e-12, -1.9114e-12,\n -2.2523e-12, 2.6415e-13, -1.5800e-12, -5.9123e-13, -2.1701e-12,\n 3.8251e-13, 1.8195e-13, -1.5011e-12, 9.5089e-13, 2.6986e-13,\n -5.9271e-13, 9.4755e-13, 3.0634e-13, 2.1217e-12, 8.4432e-13,\n -2.8487e-13, 2.7251e-13, -4.2921e-13, -2.0006e-13, 6.1866e-13,\n -1.0101e-12, 2.5467e-13, -3.8486e-13, 5.5200e-13, 1.7468e-13,\n -6.6316e-14, -7.2901e-13, 7.3921e-13, 2.8295e-13, -1.1461e-12,\n 8.6363e-13, 8.0936e-13, 1.8146e-13, -1.0127e-12, 5.6116e-14,\n -3.4232e-13, 2.8903e-13, 4.1289e-13, -8.4676e-13, 1.8262e-13,\n 1.5128e-12, -1.6991e-12, -1.9909e-12, 5.2456e-13, 8.4594e-13,\n -6.1347e-13, 2.4401e-13, 2.9537e-13, -4.9211e-15, 1.4372e-13,\n 3.2186e-13, 6.0505e-13, 3.5320e-13, -3.9942e-13, 1.3789e-13,\n 3.9485e-13, -2.6931e-13, -2.1332e-13, 4.3450e-13, 5.4673e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.4584e-07, 3.0289e-07, 2.5750e-08, 1.6791e-07, 1.4690e-08, 9.2175e-07,\n 6.8542e-06, 6.1672e-06, 1.8259e-07, 3.4789e-08, 6.2302e-08, 4.5745e-07,\n 5.1841e-08, 1.2814e-08, 4.4107e-08, 1.4283e-07, 5.8401e-07, 2.4631e-07,\n 1.4203e-06, 1.1454e-07, 4.9585e-07, 5.2661e-07, 4.2871e-08, 4.6967e-08,\n 4.9666e-07, 2.4238e-06, 1.7587e-06, 9.1531e-08, 3.1988e-06, 3.9987e-08,\n 1.9741e-07, 3.2902e-07, 2.4554e-06, 5.9063e-06, 3.5943e-08, 8.0504e-09,\n 5.9581e-07, 5.5428e-08, 2.3029e-06, 1.3948e-07, 1.2711e-08, 7.7870e-08,\n 1.7073e-06, 4.4103e-07, 1.4015e-06, 5.0921e-07, 2.2215e-07, 3.2196e-07,\n 4.6233e-08, 1.5492e-06, 9.6188e-08, 1.3274e-06, 1.8624e-08, 2.6337e-08,\n 1.8526e-08, 1.7279e-06, 3.7766e-06, 2.8183e-07, 6.7675e-06, 2.2112e-08,\n 6.5957e-07, 4.3514e-08, 9.0894e-08, 4.4474e-08, 4.8185e-06, 7.9715e-09,\n 2.4288e-06, 8.6783e-07, 1.3428e-06, 3.4973e-06, 3.2744e-07, 1.4965e-08,\n 6.9815e-08, 9.3622e-09, 1.4616e-08, 2.0755e-06, 6.9515e-09, 2.6254e-06,\n 1.2147e-06, 7.7793e-08, 3.7085e-07, 3.8785e-07, 9.1526e-09, 2.0385e-06,\n 7.5443e-09, 1.1312e-07, 1.6220e-06, 5.3812e-07, 2.0801e-06, 3.4794e-08,\n 3.1470e-07, 1.2918e-08, 1.7475e-07, 7.2266e-09, 1.2536e-06, 4.7241e-06,\n 2.9854e-08, 3.8547e-07, 3.3306e-08, 3.8624e-07, 9.0067e-08, 3.7820e-07,\n 2.3043e-08, 5.3967e-08, 5.2405e-07, 1.8029e-07, 2.7424e-08, 1.3505e-08,\n 3.4218e-06, 1.3326e-08, 5.7829e-06, 2.8101e-06, 5.7622e-06, 2.6047e-07,\n 7.6560e-09, 3.7084e-08, 3.6355e-08, 6.6370e-06, 4.4761e-08, 1.5443e-08,\n 1.2881e-06, 4.8794e-09, 1.3467e-07, 6.6824e-08, 3.9632e-09, 2.4960e-06,\n 4.3556e-07, 1.8000e-08, 2.7849e-07, 2.4102e-07, 6.7821e-08, 9.1752e-07,\n 7.4553e-07, 9.5735e-07, 2.5437e-06, 3.5322e-06, 2.0659e-07, 7.6874e-09,\n 1.3001e-07, 2.2853e-06, 1.3248e-06, 2.5978e-07, 5.1055e-08, 1.5147e-06,\n 7.1825e-09, 2.0320e-07, 1.8635e-08, 8.1176e-08, 5.9107e-06, 8.1585e-06,\n 1.8069e-08, 7.8128e-07, 4.2792e-06, 1.5323e-07, 2.2768e-07, 1.1428e-06,\n 4.4324e-07, 1.7274e-06, 6.0148e-09, 8.5993e-08, 1.2922e-07, 5.5965e-08,\n 4.8930e-06, 1.3819e-08, 1.5943e-06, 3.5502e-08, 6.0593e-08, 4.2643e-06,\n 7.3433e-08, 7.1970e-07, 1.5736e-06, 1.1630e-07, 1.1444e-06, 5.3393e-08,\n 8.8277e-07, 2.7662e-06, 6.2887e-08, 2.3493e-06, 7.1370e-09, 2.5332e-07,\n 6.0943e-08, 6.0745e-08, 2.8457e-08, 1.3165e-08, 9.8397e-09, 3.1148e-07,\n 4.3335e-06, 7.9668e-08, 3.2906e-07, 9.7270e-09, 3.0198e-07, 4.3665e-07,\n 3.8094e-06, 3.6273e-07, 6.7324e-08, 1.6474e-07, 2.0742e-08, 1.6198e-06,\n 4.2629e-06, 7.1630e-06, 8.8056e-09, 2.2773e-06, 2.2183e-06, 1.5007e-08,\n 3.9529e-06, 9.6112e-07, 1.2871e-05, 3.1546e-08, 3.2300e-08, 1.2196e-05,\n 1.1289e-06, 7.2058e-09, 4.2854e-09, 1.3813e-06, 3.8634e-06, 2.1063e-08,\n 3.1332e-06, 7.3741e-08, 2.4309e-09, 4.5044e-07, 6.4337e-08, 3.3446e-08,\n 1.6798e-07, 3.9675e-07, 1.8078e-06, 7.9224e-09, 2.9611e-07, 9.6088e-07,\n 9.6632e-08, 8.7231e-08, 5.5934e-07, 2.0881e-08, 6.9064e-07, 4.3146e-07,\n 4.8250e-06, 9.5923e-08, 8.1912e-07, 2.7831e-06, 1.3221e-08, 2.0821e-07,\n 2.3949e-08, 6.9875e-08, 1.2822e-05, 1.0076e-08, 9.3951e-09, 2.4362e-08,\n 7.4395e-09, 8.5235e-08, 5.0651e-07, 1.9508e-06, 1.2030e-06, 4.5873e-08,\n 1.0256e-08, 4.9632e-08, 9.2767e-09, 1.6552e-06, 6.4990e-07, 6.1395e-08,\n 9.6393e-09, 9.5303e-07, 6.0672e-07, 2.9955e-08, 6.9013e-07, 7.6253e-08,\n 4.1381e-06, 1.3813e-07, 4.3514e-07, 2.0163e-08, 2.2388e-06, 1.2120e-05,\n 9.4686e-08, 2.7166e-07, 1.5224e-06, 6.9403e-07, 1.6417e-06, 3.4222e-06,\n 1.7696e-08, 3.0960e-08, 1.9781e-06, 8.9433e-08, 1.0271e-07, 2.3005e-06,\n 7.0564e-08, 7.9718e-06, 1.7826e-08, 3.6591e-08, 8.4346e-09, 7.6191e-07,\n 1.4333e-06, 1.7493e-08, 1.4162e-06, 7.2509e-08, 3.9543e-08, 1.2685e-08,\n 5.2188e-08, 4.5986e-08, 2.4925e-06, 4.7023e-06, 1.3605e-08, 8.0113e-07,\n 3.8984e-07, 6.1216e-09, 2.0778e-08, 4.3910e-07, 1.9299e-07, 2.5839e-07,\n 4.5977e-09, 4.5073e-06, 4.4758e-08, 1.0266e-06, 1.5899e-08, 1.1584e-07,\n 5.5004e-06, 1.8797e-08, 1.3190e-06, 1.4632e-07, 1.1798e-08, 6.0266e-06,\n 6.7890e-09, 9.7174e-07, 7.8418e-09, 2.2111e-08, 6.6858e-07, 2.5893e-07,\n 1.6799e-06, 1.0735e-08, 8.4356e-08, 1.2157e-07, 1.5964e-07, 1.9837e-08,\n 1.6017e-07, 7.0226e-09, 9.4891e-07, 3.0409e-08, 3.0031e-06, 1.2460e-07,\n 4.9267e-06, 8.9305e-07, 8.5455e-06, 4.3448e-07, 1.2960e-08, 1.3831e-08,\n 7.3006e-07, 3.6354e-06, 2.4846e-06, 4.1856e-08, 3.5179e-07, 1.9287e-07,\n 1.1176e-07, 3.2225e-06, 3.1463e-08, 2.1723e-07, 3.9680e-08, 3.2975e-09,\n 8.9796e-08, 1.3051e-08, 9.5650e-07, 8.9309e-09, 5.8921e-08, 1.0872e-07,\n 6.6219e-08, 1.5197e-06, 4.9318e-08, 2.8433e-09, 1.9505e-06, 4.9713e-07,\n 4.2503e-08, 5.1373e-08, 2.6891e-07, 7.3841e-09, 2.3866e-07, 1.6910e-08,\n 1.3950e-08, 2.2458e-06, 1.4114e-06, 1.0745e-07, 2.6143e-06, 4.0991e-07,\n 2.4608e-08, 1.8874e-07, 3.0224e-06, 6.5192e-07, 1.0516e-06, 3.7377e-08,\n 6.2882e-09, 6.0704e-07, 2.9895e-08, 1.5938e-06, 3.8672e-06, 1.4477e-07,\n 5.7433e-07, 3.6191e-07, 1.1798e-07, 1.8995e-06, 1.6269e-08, 1.2270e-08,\n 8.1434e-07, 2.2927e-06, 1.5131e-05, 2.4954e-06, 2.2446e-08, 1.0098e-06,\n 4.4926e-06, 6.6040e-06, 3.1927e-08, 1.3118e-06, 6.6965e-08, 1.8599e-07,\n 2.3128e-07, 9.9407e-09, 6.9559e-09, 4.9656e-08, 4.5574e-08, 9.3965e-09,\n 7.9799e-08, 1.1282e-06, 6.7652e-07, 3.1857e-07, 5.1818e-07, 9.9864e-08,\n 1.1986e-06, 1.1129e-07, 2.5417e-06, 1.3570e-08, 7.4916e-07, 9.4274e-09,\n 1.4386e-08, 1.1860e-08, 1.3638e-07, 1.7280e-06, 6.4058e-09, 4.2966e-06,\n 5.7287e-07, 7.8444e-07, 1.4519e-08, 3.0655e-08, 1.3397e-06, 2.2023e-07,\n 2.6479e-08, 1.8178e-07, 1.4994e-08, 4.5422e-06, 1.9219e-06, 7.5679e-08,\n 1.8093e-06, 1.1893e-06, 2.1712e-06, 5.1029e-08, 8.7807e-07, 4.8592e-07,\n 1.0343e-08, 2.5802e-08, 1.7417e-07, 5.4338e-07, 7.5900e-09, 6.3737e-07,\n 6.7088e-07, 1.2613e-08, 4.1887e-08, 3.5508e-08, 3.9508e-06, 1.1636e-08,\n 5.2161e-08, 3.4842e-07, 1.2300e-06, 4.5943e-08, 1.5122e-07, 1.7125e-08,\n 1.2112e-06, 3.9734e-09, 6.9026e-07, 9.2686e-08, 7.8250e-07, 3.1181e-09,\n 3.9800e-06, 5.9408e-07, 3.9046e-07, 5.3569e-08, 9.2904e-09, 1.8468e-08,\n 5.0740e-08, 1.0933e-07, 3.5224e-07, 8.7185e-09, 1.3352e-07, 1.8320e-06,\n 2.1086e-06, 2.5306e-08, 8.3622e-09, 3.9966e-06, 6.9227e-07, 1.0344e-06,\n 1.0066e-06, 1.2646e-07, 4.4976e-07, 1.4203e-06, 6.1814e-08, 5.5407e-06,\n 2.3310e-06, 3.1827e-07, 2.1646e-08, 8.6825e-06, 1.5790e-07, 7.0604e-07,\n 1.5324e-06, 1.9679e-08, 2.8469e-07, 9.3320e-09, 1.7893e-08, 7.7443e-08,\n 1.4374e-07, 3.8044e-06, 1.1645e-06, 6.1126e-07, 1.1016e-05, 4.4935e-08,\n 3.9170e-07, 1.0940e-08, 2.1356e-06, 2.0022e-08, 1.4005e-06, 3.6039e-07,\n 1.0134e-06, 3.9654e-08, 2.0225e-07, 9.9518e-09, 1.6399e-08, 1.6981e-07,\n 1.6290e-07, 1.3729e-06, 1.2542e-06, 7.0370e-06, 1.5552e-06, 2.5568e-08,\n 1.5199e-08, 5.2346e-08, 2.0344e-07, 2.7408e-06, 2.9494e-08, 6.6168e-08,\n 6.3134e-08, 1.1267e-06, 1.1570e-06, 2.3503e-07, 1.2428e-07, 1.1401e-07,\n 6.6413e-09, 2.7155e-06, 1.7263e-06, 3.2476e-08, 9.2070e-06, 4.4879e-08,\n 3.6635e-07, 4.3380e-06, 1.3030e-07, 4.0539e-06, 9.8866e-09, 1.2495e-07,\n 1.8755e-08, 5.5231e-07, 3.5370e-08, 1.6500e-07, 3.2959e-06, 3.6230e-09,\n 3.5755e-06, 5.4450e-08, 1.1007e-06, 5.0306e-06, 1.1918e-08, 8.3322e-06,\n 3.1051e-07, 1.0075e-08, 3.0312e-08, 2.2569e-07, 1.3318e-06, 1.8327e-06,\n 1.4594e-06, 1.8818e-08, 3.3954e-07, 2.6417e-06, 1.3003e-06, 1.5899e-08,\n 3.0039e-06, 2.3263e-07, 7.4153e-08, 2.5621e-07, 1.2751e-07, 1.4491e-07,\n 5.7113e-07, 1.1517e-06, 6.7924e-08, 4.2859e-07, 1.1027e-07, 2.8238e-08,\n 8.9107e-09, 8.1441e-08, 1.3128e-07, 3.8034e-09, 1.3361e-08, 7.2764e-08,\n 8.4624e-09, 6.3880e-09, 1.1308e-07, 1.1535e-08, 4.7693e-07, 2.2783e-08,\n 1.3433e-07, 4.6138e-06, 1.3852e-07, 6.3293e-06, 1.0246e-08, 1.7405e-08,\n 9.1039e-07, 6.1400e-09, 1.6773e-07, 1.3294e-08, 1.4540e-06, 2.1989e-07,\n 3.1256e-07, 3.0502e-08, 1.0672e-08, 1.8145e-06, 1.6089e-07, 7.6830e-08,\n 2.4723e-06, 8.7038e-07, 2.8018e-06, 8.4232e-07, 1.0129e-07, 9.3407e-08,\n 5.6153e-08, 3.9215e-08, 4.8616e-08, 1.0171e-08, 4.0199e-08, 4.9485e-09,\n 9.4063e-09, 1.2389e-07, 1.0149e-07, 3.3291e-06], device='cuda:0')" + }, + "44": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-3.4991e-16, -1.3577e-13, -1.3115e-15, 2.9149e-16, -5.5368e-15,\n -2.3228e-15, -1.5083e-14, -1.2479e-14, 1.0307e-15, -2.8300e-16,\n -4.6237e-15, 2.6260e-16, -6.0451e-16, -4.0124e-15, 5.3541e-16,\n -1.4512e-15, -2.7739e-14, 1.7461e-15, -3.6094e-15, -2.3381e-14,\n -1.0159e-15, -1.1621e-13, -2.2404e-14, 1.0265e-14, -4.0082e-15,\n -6.0743e-14, -1.9618e-15, 3.1510e-17, -5.9168e-14, -4.7238e-15,\n 5.0175e-17, 2.0938e-14, -2.3133e-15, -5.9698e-14, 1.6426e-15,\n 4.3047e-16, -1.0384e-14, 6.8213e-16, -8.7990e-14, 9.9740e-15,\n -5.6590e-16, -1.5144e-15, -7.5438e-14, -1.8925e-14, -1.2326e-14,\n -4.4700e-15, -6.3481e-16, -1.8372e-14, -4.3903e-15, -4.6614e-15,\n 1.0368e-14, -3.2935e-16, 3.4405e-16, 7.7310e-16, -8.9286e-17,\n 6.2625e-16, -2.7044e-14, -1.9619e-14, -9.2060e-15, 2.2381e-15,\n -3.1758e-14, -1.8184e-14, 9.1006e-16, 1.0531e-15, -9.3422e-14,\n 7.5712e-15, -8.5825e-15, 1.1080e-16, -4.8706e-15, 4.3771e-16,\n -1.0475e-14, 2.1890e-16, -9.2004e-14, -1.9818e-15, -1.7364e-15,\n -1.7286e-14, -3.2200e-16, -1.1401e-16, -1.0513e-13, -2.0886e-16,\n 5.9029e-16, -4.0737e-14, 7.4129e-15, -1.0602e-15, -4.8652e-17,\n 2.5255e-15, -6.6481e-15, -1.2876e-13, -6.0959e-14, 1.6944e-15,\n 7.9453e-16, 5.9222e-15, 2.8689e-15, -1.4679e-15, -1.0971e-13,\n -3.9683e-14, -4.8110e-16, 1.6438e-15, -1.2347e-14, 3.5895e-16,\n -3.6490e-15, -9.9795e-15, -1.6805e-14, -8.5863e-15, -1.6633e-14,\n -8.9829e-15, 1.9935e-16, -5.1648e-16, 1.1110e-15, -1.0265e-15,\n -6.6654e-14, 4.1163e-15, -1.3390e-15, -3.7384e-15, 8.0657e-16,\n -5.7226e-15, -9.1353e-16, -3.4642e-15, -2.1166e-16, -5.2102e-14,\n -5.1634e-15, -1.1869e-15, -6.4967e-15, -1.0223e-15, -6.2161e-16,\n -2.0835e-15, -2.5531e-13, -4.8218e-15, 3.3119e-15, 1.7035e-15,\n -4.5507e-16, 1.7040e-15, -7.9677e-16, 3.2979e-15, -4.9897e-16,\n -4.6992e-15, -5.6677e-14, -2.0169e-14, -2.9719e-15, -1.8410e-14,\n 2.0206e-15, -1.1525e-14, 9.0610e-16, 2.2520e-14, -1.8039e-15,\n 3.1752e-17, -8.2387e-16, -1.2829e-15, -1.9152e-14, 2.3047e-14,\n 8.7102e-16, -3.7582e-14, 3.1325e-15, 1.5016e-15, -9.3317e-16,\n 3.1641e-15, 9.4258e-15, 1.8353e-15, -1.7456e-15, 2.4941e-14,\n 9.6051e-16, -4.8623e-14, -2.5417e-15, -2.3416e-16, -7.3340e-15,\n 1.6949e-15, 3.0573e-15, -2.7670e-14, -3.2656e-16, 7.1629e-15,\n -1.0975e-15, -1.1082e-15, -3.2400e-14, -7.2375e-14, -1.3541e-14,\n 4.1415e-16, 1.4804e-16, -6.2079e-16, -2.1757e-14, -6.0869e-15,\n -2.0394e-14, -1.6482e-16, 3.7312e-15, -3.2821e-16, -4.3723e-15,\n -1.7386e-15, -1.8390e-14, 2.3798e-15, -1.1741e-15, -2.7664e-15,\n 1.7832e-15, -6.3038e-15, -4.4101e-17, -1.7543e-14, 4.8086e-15,\n -7.5914e-14, 2.2300e-15, -4.5257e-15, 9.6438e-15, -4.3176e-15,\n -1.4198e-15, -1.3071e-14, -1.2315e-14, -2.6275e-15, 2.3839e-15,\n 5.0094e-14, -3.4692e-14, -6.3369e-16, 1.6326e-16, -8.6266e-15,\n -2.4861e-15, -6.3165e-16, -3.3682e-15, -8.8026e-15, 3.6990e-16,\n -5.4723e-16, -2.8589e-14, -3.0474e-16, -9.5514e-15, 1.2529e-15,\n 1.3972e-15, 5.1243e-16, -7.6286e-15, -1.3757e-14, -1.0802e-16,\n 1.9441e-15, 7.5480e-15, -1.4518e-14, 4.0734e-15, -3.5628e-14,\n -1.9328e-14, 3.3997e-16, -2.1663e-15, -3.6214e-14, -3.1948e-14,\n 3.8812e-15, -2.5197e-15, 3.5109e-14, 1.2775e-15, 6.8911e-15,\n -2.5941e-14, -4.0586e-14, -5.7265e-15, -3.4497e-15, -4.3773e-16,\n 2.4649e-15, 1.1744e-16, 3.1758e-15, 1.0910e-15, -2.5398e-14,\n 6.6292e-15, -4.5909e-15, 4.2885e-15, -4.4313e-14, 1.3186e-15,\n -4.3014e-14, -1.6367e-14, -2.1576e-15, -9.6703e-15, -1.9326e-15,\n -2.7716e-14, -3.6285e-15, -1.3662e-14, -3.7582e-15, -1.5015e-15,\n 3.8209e-15, 1.3448e-14, 5.5508e-15, 2.5046e-15, -1.0904e-13,\n 5.2253e-16, 1.2910e-15, 1.0602e-16, -4.4978e-14, 2.3120e-15,\n -5.8800e-14, -1.9355e-15, 1.2061e-15, -4.1065e-15, 1.0492e-15,\n -5.5110e-14, -3.5873e-15, -4.1357e-15, -2.2506e-14, -3.0426e-15,\n -1.5928e-13, -7.4281e-16, -1.0139e-13, -5.9887e-15, 2.5267e-14,\n 8.4963e-17, -2.1205e-14, 1.2807e-15, 6.7814e-16, -7.0680e-15,\n -1.3520e-14, -2.1514e-14, -5.8868e-15, 2.5201e-16, -4.9213e-16,\n 3.3982e-15, 7.1482e-16, 3.5168e-15, 3.1927e-15, 6.9688e-17,\n 3.3526e-14, -1.5685e-14, -8.8938e-16, -1.9584e-15, 3.6104e-15,\n 6.8910e-16, -1.7804e-16, -1.3811e-13, -5.9664e-16, 2.0107e-15,\n -2.8617e-15, -8.4330e-15, -1.9339e-14, -5.4599e-17, 1.4837e-15,\n 1.8056e-15, -3.7807e-15, -8.5739e-16, 4.4368e-15, -9.0176e-15,\n -1.2804e-14, -1.1045e-15, -4.4384e-15, -5.8689e-16, -5.7338e-15,\n 5.9447e-16, -2.7081e-16, -6.4607e-16, -4.2181e-16, -2.1132e-14,\n 4.6735e-16, 5.2796e-15, -1.8222e-14, -6.5858e-14, -3.2418e-14,\n -3.1556e-15, 1.8949e-16, -1.7386e-13, -1.7767e-14, -4.7453e-15,\n -1.0248e-13, 1.3961e-17, 1.8621e-14, -2.4398e-13, -1.4064e-15,\n -4.0800e-16, 4.3429e-16, -1.5181e-14, 4.5333e-15, 1.8603e-14,\n 2.5613e-15, -6.7272e-14, -1.7751e-16, 7.7870e-16, -8.0230e-15,\n -7.4379e-16, 1.0897e-15, -3.8772e-15, 3.1099e-15, -3.0682e-16,\n 5.8954e-16, -1.2124e-13, -4.7914e-14, -1.5498e-15, -1.9619e-16,\n -1.1675e-15, 1.3800e-15, -5.2486e-14, -2.6783e-15, -4.3668e-15,\n -1.7796e-14, 1.7041e-15, 2.4092e-15, -1.4482e-14, -2.4610e-15,\n -6.1978e-14, -3.7667e-15, 9.0862e-17, -1.1410e-17, 3.7497e-16,\n 2.1906e-15, -2.5668e-14, 1.9833e-15, -1.6730e-14, 1.0520e-15,\n 5.9080e-15, 3.1649e-15, -1.2142e-15, 1.9143e-14, -1.9617e-15,\n 2.3556e-15, -1.9289e-14, -6.2254e-14, -2.3716e-14, 2.4813e-15,\n -2.9284e-15, -2.0130e-14, -1.5210e-14, 4.9470e-16, -5.2508e-16,\n 4.4094e-16, 2.9229e-16, -1.0425e-14, 3.1561e-16, -8.1540e-15,\n -2.5542e-14, -3.1800e-15, -1.1887e-16, -6.7316e-16, 2.6552e-15,\n -7.3688e-15, 2.8870e-15, 1.5376e-14, -1.1467e-13, 1.2695e-15,\n -1.2869e-15, 4.1058e-15, -2.8120e-14, -6.9216e-16, -1.8821e-15,\n -1.1963e-15, -1.0426e-13, 7.5142e-16, -2.1954e-15, 8.8993e-16,\n -2.9297e-16, 1.6565e-15, -7.8953e-15, -1.4114e-14, -7.9048e-14,\n -1.1719e-15, -1.6458e-14, -3.2681e-14, -3.7479e-15, -8.0976e-14,\n -1.9742e-15, -6.8585e-17, -3.1396e-14, 4.0932e-16, -1.0036e-14,\n -1.0292e-14, -3.0538e-14, 8.2742e-17, -1.3898e-14, -5.1878e-14,\n -8.0139e-15, -4.7832e-14, -1.3390e-13, -6.8473e-14, 8.2322e-15,\n -1.4774e-14, 3.3716e-15, 5.7812e-16, -1.0427e-15, -4.9448e-14,\n -3.6047e-15, 3.3395e-16, -1.1325e-14, -8.8597e-14, -1.9425e-14,\n -2.0407e-15, 1.1191e-15, -5.1015e-15, -1.3837e-16, -5.7033e-16,\n -4.4648e-15, -4.0961e-16, -5.7452e-15, -7.2041e-16, 3.9062e-15,\n -2.5399e-15, -1.9371e-14, -3.8446e-15, 3.2385e-16, -9.2238e-15,\n -8.2903e-16, -1.0332e-14, 1.4242e-15, 7.5244e-16, -3.7485e-16,\n -3.9132e-15, 4.1321e-15, -8.7005e-14, -1.5417e-15, -5.8008e-14,\n 1.8052e-16, -1.1844e-14, -2.6304e-14, -5.9359e-15, 1.3954e-15,\n -3.5370e-14, 3.2085e-15, -1.8920e-15, 8.6328e-16, -9.5459e-16,\n -1.0427e-15, -1.8101e-15, -1.4876e-15, -4.6388e-15, -1.3191e-15,\n -6.6777e-16, -6.1563e-14, -1.9715e-15, -9.5044e-15, -8.8965e-16,\n -1.3633e-14, -4.1233e-15, -5.1179e-15, 1.8632e-15, -1.3031e-14,\n -6.3981e-16, -1.9321e-14, -9.5936e-15, 1.5603e-15, 1.4353e-17,\n 3.0291e-15, -5.0609e-16, 7.2744e-16, -6.0647e-16, 8.8124e-15,\n -8.5548e-15, 3.0651e-15, -3.6292e-16, 8.2458e-16, -1.0606e-14,\n 2.1068e-15, -2.4430e-17, -1.4090e-15, -2.4532e-16, -2.3810e-15,\n -6.4559e-15, -2.3362e-15, 8.9820e-15, 3.0333e-15, -1.2270e-13,\n -5.6287e-14, -3.8396e-14, 2.3338e-15, -9.2255e-16, -3.5021e-14,\n 1.7740e-15, -2.2453e-16, 1.3732e-14, -8.9805e-15, 4.7965e-15,\n 1.1426e-15, 1.3398e-15, 6.2687e-16, -2.5740e-14, -5.2628e-16,\n 4.5349e-15, -4.1220e-15, 4.5933e-15, 1.3722e-15, 1.6376e-15,\n 1.3293e-14, -4.7484e-16, -1.6231e-14, -3.1338e-16, -8.4237e-17,\n -1.1190e-14, -8.9436e-15, -2.5825e-15, 4.0864e-16, -4.0863e-14,\n -1.6723e-15, 1.1608e-15, 2.7679e-15, -3.1168e-15, -7.4585e-14,\n -2.9332e-15, -4.0802e-14, -7.1449e-16, -1.4023e-13, -5.8565e-14,\n -6.5597e-14, -1.0686e-15, -4.6492e-14, -5.0522e-14, -6.9114e-14,\n 2.3355e-16, -3.0687e-15, -4.2871e-14, 2.5619e-15, -4.3840e-16,\n -2.7813e-16, 1.6400e-15, -1.0628e-14, 1.7342e-14, 3.9614e-15,\n -1.0038e-14, -7.8168e-16, -3.5024e-15, -5.9372e-16, -1.5326e-15,\n -1.0686e-15, 1.9324e-17, -2.5073e-14, -1.5551e-15, 8.2151e-16,\n -6.8173e-14, -1.2913e-15, -6.5681e-15, 3.2301e-16, -3.2188e-14,\n -1.9136e-15, 4.9689e-16, -4.5402e-15, -1.0623e-15, 2.0568e-15,\n 1.3220e-16, -1.2761e-15, 1.7308e-15, -2.5300e-14, -2.3214e-15,\n 1.2274e-14, -2.1890e-14, -1.9695e-13, 7.3253e-15, 4.2645e-15,\n -1.8088e-14, 6.6859e-16, -5.8581e-16, 2.6810e-15, -4.3348e-16,\n 2.2127e-16, -1.9601e-15, 3.1555e-16, 8.7793e-15, -1.5338e-15,\n 2.3363e-15, -6.0082e-15, -1.7911e-13, -7.9843e-14, 2.4369e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.9445e-12, 8.5587e-12, 2.7591e-13, 4.1159e-12, 6.1643e-13, 2.8979e-10,\n 1.3573e-09, 1.2393e-09, 2.2355e-11, 2.7297e-12, 1.0218e-11, 3.0992e-12,\n 1.3057e-12, 1.0916e-13, 4.2805e-12, 1.8411e-12, 4.0791e-12, 3.3448e-12,\n 5.3822e-11, 8.5143e-13, 5.3920e-11, 7.9487e-11, 6.5373e-13, 5.3656e-12,\n 1.9875e-12, 1.9461e-09, 1.1225e-10, 6.0385e-13, 1.6070e-09, 2.8091e-13,\n 1.3154e-12, 8.0126e-12, 1.3125e-09, 9.3070e-10, 2.1505e-12, 2.3927e-14,\n 9.1674e-12, 9.1788e-13, 2.1358e-09, 3.2972e-11, 3.8855e-13, 1.5468e-11,\n 7.0630e-10, 4.5495e-11, 2.1050e-10, 1.5370e-11, 9.6640e-12, 1.4377e-10,\n 5.1516e-12, 2.2629e-10, 1.3765e-12, 7.4494e-11, 1.1288e-13, 3.4535e-13,\n 2.3706e-13, 1.4887e-10, 1.5687e-09, 3.4392e-11, 5.6439e-10, 2.1965e-13,\n 2.5872e-11, 9.8123e-13, 1.8575e-11, 5.5101e-12, 9.0799e-10, 1.6868e-13,\n 8.8383e-10, 5.1299e-12, 2.9272e-11, 7.0119e-10, 1.4579e-12, 1.7794e-13,\n 2.9891e-11, 1.0061e-13, 6.4714e-13, 1.7613e-09, 1.6716e-11, 5.6096e-10,\n 2.9172e-10, 4.1796e-12, 3.1432e-11, 5.0789e-11, 1.4676e-13, 4.9351e-10,\n 6.8350e-12, 1.1972e-12, 6.6615e-10, 1.2933e-10, 6.9649e-10, 3.1539e-12,\n 1.8784e-10, 3.1953e-13, 4.9962e-13, 1.4220e-11, 1.3123e-10, 3.0260e-09,\n 2.3346e-12, 1.3242e-12, 3.7485e-13, 2.4015e-12, 3.1352e-11, 2.4180e-12,\n 6.9220e-13, 5.8691e-12, 3.2601e-11, 3.7001e-12, 2.0495e-12, 2.1223e-13,\n 2.0721e-09, 1.8869e-12, 5.1225e-09, 2.1543e-09, 1.0902e-09, 6.7324e-11,\n 2.8158e-13, 8.5255e-13, 4.3949e-12, 3.4388e-10, 6.8244e-13, 1.2459e-13,\n 6.3231e-10, 8.9562e-12, 1.5928e-11, 1.0377e-12, 5.4077e-12, 1.1354e-09,\n 1.2259e-10, 9.2465e-13, 1.2902e-11, 5.3965e-12, 4.9450e-12, 2.6673e-10,\n 5.2125e-12, 1.6666e-11, 3.9335e-11, 1.2075e-10, 1.4453e-11, 1.2740e-12,\n 2.3824e-12, 5.0641e-11, 7.9251e-11, 1.0986e-10, 2.7356e-12, 1.0859e-09,\n 2.5761e-13, 2.0656e-11, 1.2156e-13, 1.3958e-11, 1.3185e-09, 9.0820e-10,\n 9.8058e-13, 3.1939e-10, 2.7984e-10, 9.7285e-12, 2.2605e-11, 2.5786e-11,\n 5.8939e-11, 3.5212e-10, 5.4325e-14, 2.7003e-12, 1.3509e-12, 6.8716e-13,\n 2.0901e-09, 2.4859e-13, 1.6741e-10, 6.0146e-13, 3.9942e-13, 4.4600e-10,\n 9.9045e-13, 1.0397e-11, 8.3139e-10, 3.4655e-11, 2.2663e-10, 8.3056e-12,\n 6.1359e-12, 5.9316e-11, 9.2206e-12, 1.1127e-10, 1.6537e-14, 1.4707e-12,\n 1.0926e-12, 1.6302e-12, 8.4170e-13, 3.3012e-13, 3.1982e-13, 4.3516e-11,\n 4.9722e-09, 1.5001e-11, 2.1792e-12, 2.4430e-13, 1.6211e-12, 3.0926e-11,\n 7.0581e-10, 1.0495e-10, 1.0994e-11, 1.5928e-11, 4.7018e-13, 3.7681e-10,\n 2.0355e-10, 4.1273e-09, 1.1273e-13, 1.5321e-10, 3.5827e-11, 2.0818e-11,\n 3.9165e-10, 1.5671e-11, 3.5802e-09, 9.9845e-13, 1.9076e-12, 2.3790e-09,\n 1.0541e-10, 3.6978e-13, 1.6833e-12, 1.4064e-09, 1.0584e-10, 2.7040e-13,\n 1.1304e-09, 1.3909e-11, 4.9559e-12, 3.0889e-12, 3.6441e-12, 2.9660e-12,\n 1.0402e-12, 8.7332e-12, 5.7573e-10, 8.6819e-14, 3.1435e-12, 6.3206e-11,\n 6.0816e-12, 1.8167e-11, 4.3260e-11, 2.0033e-13, 7.4420e-12, 2.5867e-11,\n 5.5975e-10, 2.6062e-11, 1.0438e-10, 5.3431e-10, 4.1442e-14, 4.9961e-12,\n 2.1198e-13, 1.6376e-12, 3.4509e-09, 1.6149e-13, 9.3174e-14, 6.7564e-13,\n 6.2231e-13, 1.1638e-12, 2.8925e-11, 8.9020e-10, 2.0506e-11, 5.4996e-12,\n 3.1490e-13, 8.2658e-12, 1.3836e-13, 7.1972e-10, 1.4992e-11, 1.7534e-12,\n 3.1648e-13, 8.0592e-12, 1.0464e-11, 2.1634e-12, 2.5980e-10, 7.5600e-13,\n 6.4702e-10, 9.7750e-12, 6.4983e-12, 7.7011e-13, 3.6463e-10, 3.0036e-09,\n 7.4400e-13, 8.8975e-11, 1.1945e-10, 1.7342e-10, 5.7282e-11, 1.4963e-09,\n 2.3316e-13, 3.1186e-13, 5.2549e-11, 5.9565e-12, 7.8159e-12, 6.7633e-11,\n 5.1036e-13, 1.5882e-09, 9.3853e-13, 5.3014e-13, 4.9201e-14, 2.0019e-10,\n 2.1610e-11, 2.4801e-12, 6.6172e-11, 6.1946e-13, 2.2828e-12, 4.3935e-13,\n 7.5840e-13, 4.9807e-12, 6.8751e-11, 8.3053e-10, 6.0360e-14, 1.7031e-10,\n 1.9987e-12, 2.3177e-12, 3.3836e-11, 3.1207e-12, 3.6799e-12, 4.7160e-12,\n 2.3316e-12, 1.0381e-09, 1.0062e-13, 3.6331e-10, 9.0413e-13, 3.3491e-11,\n 5.7340e-09, 1.5436e-12, 6.8098e-12, 1.7849e-10, 9.0654e-13, 4.6429e-09,\n 1.3639e-11, 1.1776e-11, 5.8307e-14, 1.8678e-13, 1.1468e-10, 1.7152e-12,\n 6.1966e-11, 1.5475e-12, 1.2201e-12, 2.0477e-11, 1.0793e-12, 3.1304e-11,\n 2.7529e-11, 5.1267e-13, 2.4335e-11, 8.5700e-13, 1.3618e-10, 6.9506e-12,\n 2.7156e-09, 9.8111e-11, 2.4140e-09, 1.6250e-12, 2.3264e-11, 8.5725e-13,\n 8.1385e-10, 1.8344e-09, 2.0279e-10, 6.3377e-13, 9.4853e-12, 3.3338e-11,\n 6.0026e-11, 4.9961e-10, 1.4630e-13, 1.0086e-10, 1.2591e-12, 1.0779e-13,\n 7.9579e-12, 2.7546e-13, 7.7095e-12, 1.7737e-11, 2.3750e-11, 7.7707e-13,\n 9.7686e-12, 4.8325e-11, 3.7999e-12, 3.9509e-12, 8.8899e-11, 6.6546e-12,\n 9.8755e-13, 2.4034e-12, 2.7471e-12, 9.5411e-12, 3.7212e-12, 7.4775e-13,\n 2.8572e-13, 1.4845e-10, 7.0580e-11, 2.4672e-11, 3.6367e-10, 2.4530e-11,\n 1.0167e-12, 5.6932e-11, 2.7376e-09, 1.2298e-10, 7.2006e-11, 1.1155e-13,\n 5.3623e-14, 1.1741e-11, 7.8835e-13, 2.4989e-10, 1.6099e-09, 3.5082e-12,\n 5.5848e-10, 1.0157e-11, 2.3797e-12, 3.7709e-11, 1.6545e-13, 3.6615e-13,\n 9.5829e-10, 1.1104e-09, 3.7486e-09, 5.1776e-10, 2.0816e-12, 2.0697e-11,\n 2.1640e-10, 3.2064e-10, 1.3291e-13, 5.0461e-10, 1.5546e-12, 2.1294e-11,\n 6.7972e-12, 3.7602e-13, 6.6615e-14, 5.1048e-12, 5.4014e-12, 4.0268e-13,\n 7.8764e-13, 7.2669e-12, 2.8725e-10, 1.1926e-12, 1.0360e-11, 4.9911e-13,\n 6.7610e-10, 1.1245e-12, 6.3869e-10, 6.8712e-13, 2.3972e-11, 4.8167e-13,\n 4.9887e-12, 1.3075e-11, 1.9721e-12, 2.0601e-10, 2.2790e-13, 8.7658e-10,\n 1.0388e-11, 4.4534e-10, 3.7608e-13, 1.0147e-12, 1.0188e-09, 6.6373e-11,\n 5.7503e-13, 7.1632e-11, 9.8685e-13, 6.3559e-10, 1.2813e-09, 1.1536e-11,\n 7.1459e-10, 3.5949e-10, 6.9291e-11, 6.4547e-12, 3.7708e-10, 3.7052e-11,\n 2.2484e-12, 1.5244e-13, 5.5308e-11, 8.8060e-11, 4.1892e-13, 1.6008e-10,\n 1.3552e-10, 2.3202e-13, 4.6476e-12, 1.2897e-12, 1.1240e-09, 4.9896e-13,\n 6.7729e-12, 3.4224e-12, 2.3130e-11, 1.0207e-12, 9.1466e-13, 2.9723e-13,\n 2.3350e-10, 1.1898e-13, 4.8896e-11, 1.4447e-11, 1.0735e-10, 3.5594e-12,\n 2.7722e-10, 3.6785e-12, 3.0749e-10, 1.4992e-12, 4.7953e-14, 8.4674e-13,\n 7.2221e-12, 2.7346e-11, 2.6033e-12, 4.8237e-13, 6.0319e-12, 2.5206e-11,\n 6.9529e-10, 7.5004e-13, 1.0371e-13, 1.2837e-09, 2.8262e-12, 1.1750e-11,\n 2.2614e-10, 7.4097e-13, 3.7977e-12, 8.6731e-11, 9.2034e-13, 1.0200e-09,\n 2.8492e-10, 1.6952e-11, 1.9221e-12, 3.3304e-09, 1.0181e-10, 1.6614e-11,\n 3.2832e-10, 2.6616e-13, 1.2849e-10, 1.2205e-11, 3.8383e-13, 4.6819e-13,\n 1.2277e-12, 1.3729e-09, 6.1110e-10, 4.8701e-12, 8.6669e-10, 5.3507e-12,\n 1.8269e-10, 3.3778e-13, 2.7440e-11, 8.4385e-12, 1.2449e-11, 1.2665e-11,\n 2.0213e-10, 3.9270e-12, 1.8056e-12, 8.8587e-13, 4.3654e-13, 1.4340e-12,\n 2.3038e-12, 2.5739e-10, 6.6859e-11, 2.4390e-10, 4.6417e-11, 1.1093e-12,\n 2.0511e-12, 4.7292e-12, 3.5842e-12, 5.4011e-11, 2.7191e-13, 3.2347e-12,\n 2.1209e-11, 6.8501e-11, 8.9504e-11, 3.5357e-11, 7.8600e-13, 1.1870e-11,\n 1.3171e-12, 2.6002e-10, 2.8310e-10, 2.8040e-12, 3.3171e-10, 5.4751e-13,\n 1.2009e-12, 6.3286e-10, 2.5702e-11, 8.1452e-11, 1.7911e-13, 3.3144e-12,\n 9.3827e-13, 2.3786e-11, 1.0254e-12, 2.5124e-12, 1.0795e-09, 3.0143e-12,\n 1.0854e-10, 5.5775e-13, 3.5123e-10, 2.4499e-10, 4.0172e-13, 1.4412e-09,\n 2.9159e-12, 3.1207e-14, 3.4758e-13, 1.6249e-12, 1.3355e-11, 7.2645e-10,\n 6.4751e-10, 1.6343e-13, 1.6471e-10, 2.0607e-11, 4.7303e-10, 2.5579e-13,\n 2.7839e-10, 7.6151e-12, 1.5047e-12, 7.4503e-12, 1.2036e-11, 1.1364e-12,\n 6.7347e-12, 1.6600e-10, 1.1948e-11, 1.0256e-10, 6.4158e-12, 2.3874e-12,\n 6.3057e-13, 3.5365e-12, 9.8076e-12, 7.2971e-12, 5.1271e-13, 7.8757e-12,\n 1.6133e-11, 1.5894e-13, 6.1695e-13, 6.4384e-14, 1.9019e-12, 1.1129e-12,\n 1.2699e-12, 1.6102e-09, 7.1564e-12, 3.8699e-09, 3.8699e-14, 9.3431e-14,\n 3.6813e-10, 1.6811e-11, 3.5827e-12, 1.8152e-13, 5.9374e-11, 5.6271e-12,\n 5.6078e-11, 3.6662e-12, 1.8874e-13, 4.1254e-11, 3.9700e-11, 6.1673e-12,\n 2.6191e-10, 1.7377e-11, 2.0323e-10, 3.1338e-12, 1.9021e-11, 9.2606e-13,\n 2.4571e-12, 1.5220e-12, 7.1779e-12, 1.5607e-13, 1.5813e-12, 1.5083e-12,\n 2.2336e-12, 4.0744e-11, 1.9085e-11, 2.1757e-10], device='cuda:0')" + }, + "45": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.8547e-15, -1.4907e-13, 5.4033e-15, 4.0993e-15, -2.9425e-14,\n -1.6693e-14, -2.4452e-14, -1.5823e-14, -3.9325e-15, 1.3164e-16,\n 2.2931e-15, 1.2584e-15, 3.9477e-16, -1.0178e-14, -2.4962e-16,\n -6.6312e-15, -6.0026e-14, 1.1490e-14, 8.2770e-15, -5.2802e-14,\n -3.5562e-15, -7.3257e-14, -1.4068e-14, -5.8398e-15, -2.0181e-14,\n -4.6039e-14, -1.9186e-14, 1.1335e-14, -5.9554e-14, -1.4742e-14,\n 2.0032e-14, 1.8058e-14, -7.7210e-15, -3.5351e-14, 2.4418e-15,\n -1.0113e-15, -2.9673e-14, 2.1228e-15, -9.3411e-14, 1.0612e-14,\n 2.5770e-15, 1.3493e-15, -4.8606e-14, -2.0286e-14, -3.5491e-14,\n -1.8385e-14, -1.4616e-14, -1.8171e-14, 2.7817e-15, -1.0928e-14,\n 1.2064e-14, -1.8308e-16, -5.0685e-15, -3.9555e-15, 7.2515e-16,\n -1.1922e-15, -5.0101e-14, -3.1809e-14, -1.7995e-14, -4.8296e-15,\n -5.8244e-14, -3.2251e-14, -4.1930e-16, -1.3204e-15, -8.2086e-14,\n 8.3265e-15, -4.9400e-14, 1.8108e-14, -1.7610e-14, -8.8760e-16,\n -2.1975e-14, -3.6786e-15, -7.6338e-14, 2.1615e-15, 1.1756e-15,\n -2.6599e-14, 9.5446e-17, -1.2552e-15, -1.2733e-13, 1.4542e-15,\n 3.6944e-15, -3.9943e-14, -5.9853e-15, -9.7361e-15, 1.5599e-17,\n -6.7719e-15, -2.7363e-14, -1.0380e-13, -4.2660e-14, -7.8902e-15,\n 4.7367e-16, 8.1980e-15, -1.6393e-14, 5.8349e-16, -8.0107e-14,\n -6.2323e-14, -2.9468e-15, -4.0668e-15, -1.2451e-14, -2.4496e-15,\n -3.6719e-14, -1.9085e-14, -3.0954e-14, -1.0798e-14, -2.3024e-14,\n -3.0389e-14, -5.0473e-17, 5.0850e-16, -2.8692e-15, 3.4862e-16,\n -9.3204e-14, -2.0104e-14, -1.1995e-14, -1.4754e-14, -9.9919e-16,\n -7.9656e-15, 2.4088e-15, 5.2740e-15, -1.9523e-14, -7.2433e-14,\n -2.1128e-14, 5.5934e-16, -4.5029e-15, -1.7109e-14, 2.2043e-16,\n -3.1906e-14, -1.3505e-13, 1.8352e-15, -1.0261e-14, -4.9713e-16,\n 1.3242e-15, -7.3366e-15, 1.4468e-14, -1.2781e-14, -7.6220e-15,\n -2.2651e-15, -4.6014e-14, -2.1894e-14, -1.0308e-14, -2.6715e-14,\n -1.2378e-14, -2.5702e-14, 7.9185e-15, 2.8714e-14, 1.5692e-15,\n 1.1734e-16, 1.6729e-16, 3.3724e-15, -2.0924e-14, 3.2976e-14,\n -1.2777e-15, -3.8527e-14, 9.3716e-16, -1.4996e-14, 4.5167e-15,\n 8.5042e-15, 1.4416e-14, -9.1876e-15, -6.6708e-15, 4.1054e-14,\n 6.1331e-15, -2.5100e-14, -5.5799e-14, 3.4868e-16, -6.8844e-15,\n 3.5128e-15, -3.5519e-15, -2.6858e-14, -4.6955e-15, 6.1008e-15,\n -1.2731e-15, 8.6836e-16, -4.2234e-14, -4.7752e-14, -4.1559e-14,\n 3.6801e-16, -1.6187e-15, 1.6728e-15, -2.2196e-14, -1.3464e-14,\n -1.6395e-14, 1.6814e-15, -1.3141e-14, 8.9696e-15, -2.6649e-14,\n 1.5320e-14, -1.3706e-14, -2.5078e-15, -7.7319e-15, 1.6009e-14,\n -2.6491e-15, -1.2688e-14, -5.6909e-15, -9.8161e-15, 2.2987e-14,\n -5.5148e-14, -9.0018e-16, -5.6323e-16, 9.6239e-15, -8.9562e-15,\n -2.8679e-14, -3.0275e-14, -1.8621e-14, 1.2682e-15, 1.4683e-15,\n 4.1092e-14, -3.1411e-14, 2.1112e-16, 3.3144e-15, -1.7804e-14,\n -1.1058e-14, 1.1431e-15, 2.5906e-15, -1.8428e-14, 7.0465e-17,\n -3.0788e-16, -4.8853e-14, 3.1843e-16, -1.4128e-14, -4.5726e-16,\n -2.4907e-15, -7.2805e-16, -8.1586e-16, -1.5998e-14, 4.4509e-15,\n -9.8094e-17, 1.4344e-14, -4.8651e-14, -1.1062e-14, -3.2790e-14,\n -2.1252e-14, -1.8102e-15, 3.7086e-15, -5.8123e-14, -3.6851e-14,\n -2.2457e-15, -1.1347e-14, 4.0846e-14, -2.9840e-14, 1.5348e-14,\n -3.8950e-14, -2.7407e-14, -1.2755e-14, -4.2002e-16, 2.9016e-15,\n -8.1863e-16, -8.7499e-17, 9.8670e-15, -8.1518e-15, -4.2175e-14,\n -5.6717e-15, 1.2716e-14, -4.2984e-15, -5.0995e-14, -6.1852e-15,\n -3.6912e-14, -4.0139e-14, -5.2208e-15, 6.9268e-15, -9.4871e-15,\n -4.7171e-14, 6.7798e-15, -2.0802e-14, 3.2412e-15, -2.2446e-15,\n 1.6342e-15, 2.4123e-14, 2.0239e-16, 5.8441e-15, -1.1378e-13,\n -2.8320e-16, -3.2419e-15, 3.4877e-15, -2.3613e-14, 8.6190e-15,\n -6.1371e-14, -1.5623e-14, -2.1316e-16, -1.2603e-14, 5.4031e-16,\n -4.0178e-14, 7.6398e-15, -1.5126e-14, -3.1935e-14, 2.5739e-15,\n -1.3696e-13, 3.1490e-16, -7.0527e-14, -2.5159e-14, 2.0388e-14,\n -7.6128e-15, -4.5570e-14, -1.5052e-15, 1.4183e-15, -1.8997e-14,\n 1.1737e-14, -5.0088e-14, -4.1518e-14, -3.1913e-15, -1.6551e-15,\n 1.8833e-15, -1.6759e-14, -1.8174e-15, -1.2304e-14, -7.4460e-16,\n 2.9191e-14, -1.6526e-14, -6.1889e-15, -1.4595e-14, -3.1302e-15,\n -1.0047e-14, 1.0538e-16, -9.0780e-14, -1.1868e-16, -6.8116e-15,\n 2.7487e-15, -4.5032e-14, -2.5390e-14, 4.8483e-18, 1.7336e-15,\n -8.8721e-15, -6.2685e-15, -1.8307e-15, 1.6594e-14, -1.9465e-14,\n 7.3644e-15, -8.1257e-15, -1.0412e-14, 3.6402e-15, 2.8699e-15,\n -1.2953e-14, 8.0034e-16, -6.0508e-15, -1.7507e-15, -4.0396e-14,\n 3.0204e-16, 2.6807e-15, -3.4876e-14, -7.9423e-14, -4.0547e-14,\n 1.3885e-15, 8.1483e-16, -7.9647e-14, -2.5895e-14, -2.9609e-14,\n -9.3353e-14, 1.2852e-16, 2.1267e-14, -6.2894e-14, -8.9688e-15,\n -4.6223e-15, -5.7904e-16, -6.9363e-15, -1.9749e-14, 2.4686e-14,\n -1.3376e-14, -9.4758e-14, -1.0013e-16, -2.6311e-15, -2.2007e-14,\n 9.7984e-16, -5.3971e-15, 3.4289e-15, -7.0848e-15, 1.6722e-15,\n -4.7399e-15, -5.7804e-14, -2.1094e-14, -1.7687e-14, 2.2568e-17,\n 8.9654e-16, 2.0458e-15, -5.0507e-14, -3.8964e-15, -1.6926e-14,\n -1.6152e-14, -1.1344e-15, -1.7586e-15, -2.2911e-14, 2.2719e-15,\n -4.9143e-14, 7.5869e-15, 1.6805e-14, -6.6279e-16, -3.1103e-15,\n -1.4086e-15, -4.7440e-14, 2.9006e-15, -4.8669e-14, -8.8720e-15,\n 6.6612e-15, -2.6857e-15, -2.1186e-15, 4.5229e-14, -1.6994e-14,\n -7.9586e-15, -3.3086e-14, -7.4398e-14, -8.7530e-14, -1.4682e-14,\n 2.5821e-15, -2.4376e-14, -2.7517e-14, 1.0841e-14, -1.2411e-14,\n -3.0867e-14, 5.9227e-15, -2.4859e-14, -1.7509e-14, -1.7650e-14,\n -4.0439e-14, 2.4943e-15, -9.1451e-15, 1.4108e-15, -1.7023e-14,\n -3.3931e-14, 7.1511e-15, 2.0899e-14, -7.6393e-14, -2.4119e-14,\n 3.8427e-15, -6.3844e-15, -4.4545e-14, -2.3264e-15, -1.3898e-14,\n 1.2748e-15, -6.0761e-14, -6.6104e-16, -2.1861e-15, -1.1905e-14,\n 6.5196e-16, 4.0299e-15, -3.6617e-14, -2.3600e-14, -6.7196e-14,\n -3.4617e-14, -2.9617e-14, -2.3876e-14, 1.6637e-15, -3.9252e-14,\n 8.2213e-16, -7.3476e-15, -4.4537e-14, -7.4717e-16, -1.4928e-14,\n -1.3811e-14, -3.9657e-14, -5.4868e-17, -2.1085e-14, -5.4240e-14,\n 5.0003e-15, -4.0068e-14, -4.3739e-14, -5.9677e-14, 3.3488e-14,\n -2.7932e-14, -1.1663e-14, -3.0881e-15, 7.4957e-16, -3.6303e-14,\n -5.9423e-15, -4.2353e-15, 7.8807e-15, -1.0323e-13, -6.0752e-14,\n 1.8995e-14, -1.5840e-15, 2.6283e-15, -1.4018e-14, -4.3362e-18,\n -1.0562e-14, 6.3683e-16, -1.5918e-14, 4.7891e-16, 2.1450e-14,\n -3.1054e-14, -1.9997e-14, -4.2376e-15, 8.0910e-15, -1.2239e-14,\n 2.6947e-16, -8.2391e-15, 6.6942e-15, -2.4665e-15, -3.8374e-16,\n -1.1694e-14, 2.0036e-15, -7.4031e-14, 1.7235e-15, -9.0058e-14,\n 2.7147e-14, -3.4996e-14, -2.7582e-14, 4.5246e-15, 2.1956e-15,\n -6.5600e-14, 6.4521e-15, -2.8632e-15, -4.2179e-15, -1.2976e-15,\n 5.1830e-16, -4.7020e-15, -8.6100e-15, -1.9269e-14, -4.6425e-15,\n -5.5330e-15, -6.8714e-14, 8.4043e-16, -1.2425e-14, -6.9328e-15,\n -2.1802e-14, -5.8378e-15, 1.8259e-15, -4.4243e-15, -1.3862e-14,\n 4.0600e-16, -3.2267e-14, 5.8499e-15, -1.5091e-16, 6.0512e-17,\n -1.5272e-14, 4.2830e-15, -2.7468e-17, 1.6319e-16, -8.8759e-15,\n 5.2858e-15, -2.9131e-15, -8.3619e-15, -1.4781e-14, -3.8955e-14,\n -1.1921e-14, 1.3886e-14, -1.4586e-14, 1.5658e-15, -9.9812e-15,\n -6.8387e-15, -2.2413e-15, 3.0959e-14, 3.4140e-15, -7.5573e-14,\n -4.2695e-14, -5.5060e-14, 2.9219e-15, -1.3243e-14, -5.8167e-14,\n -3.2856e-14, -4.4605e-15, 1.5316e-14, -1.1259e-14, 1.3864e-14,\n -1.8658e-14, -1.9098e-15, -3.4642e-15, -4.2982e-14, 4.5265e-16,\n 1.6177e-14, -8.9050e-15, -3.3456e-15, 6.5117e-16, -6.1254e-15,\n 9.6540e-15, -1.5949e-14, -2.7853e-14, 1.4620e-16, -9.8364e-17,\n -3.5228e-14, -2.6148e-14, -4.1090e-14, -6.8043e-16, -5.7734e-14,\n -9.2559e-15, -3.1385e-15, 6.3824e-15, -2.3233e-14, -1.2186e-13,\n 4.9245e-18, -3.9624e-14, 5.9756e-16, -7.4004e-14, -6.9853e-14,\n -8.5686e-14, 8.0229e-16, -6.6520e-14, -4.7397e-14, -8.2408e-14,\n -1.6896e-15, 7.8784e-16, -6.4207e-14, 8.1216e-15, -6.4105e-15,\n 1.8834e-16, 4.1503e-15, -1.5322e-14, 3.8647e-14, 6.4659e-15,\n -1.4463e-14, -2.5264e-15, 1.4818e-15, -1.6616e-14, 5.3644e-15,\n 2.8548e-16, -4.2140e-15, -3.3481e-14, 2.1811e-15, -7.4318e-15,\n -3.7975e-14, -3.6988e-14, -6.8762e-15, -3.7953e-15, -5.3672e-14,\n 1.1715e-14, 7.2773e-15, -1.1470e-14, 3.3941e-16, -1.0722e-14,\n -2.3365e-14, 2.3020e-17, 5.8489e-16, -4.4307e-14, 2.5122e-15,\n 1.6984e-14, -6.3028e-14, -9.1636e-14, -2.5524e-16, 2.3059e-15,\n -3.6564e-14, 1.3484e-15, 9.2496e-16, -1.7776e-15, -1.8828e-15,\n -8.5845e-18, 4.6942e-15, 8.7704e-16, -2.6208e-14, 1.0145e-15,\n -4.1605e-15, -2.9434e-14, -5.6709e-14, -3.2584e-14, -2.7415e-16],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.6548e-10, 5.3969e-11, 6.7670e-12, 4.4953e-11, 3.4109e-12, 2.0884e-10,\n 1.4321e-09, 1.2533e-09, 4.2408e-11, 1.1166e-12, 3.9472e-12, 9.9220e-11,\n 1.5714e-11, 1.9594e-12, 1.1555e-12, 2.6436e-11, 1.0004e-10, 6.1648e-11,\n 3.1160e-10, 3.0279e-11, 1.2040e-10, 1.0469e-10, 7.4961e-12, 1.5646e-12,\n 1.0994e-10, 5.3009e-10, 3.7869e-10, 1.2733e-11, 6.4323e-10, 3.8542e-12,\n 5.1374e-11, 7.6684e-11, 5.1015e-10, 1.1897e-09, 1.0317e-11, 2.9109e-13,\n 1.4008e-10, 1.1366e-11, 5.2342e-10, 3.5463e-11, 6.8113e-13, 5.5343e-12,\n 3.6083e-10, 8.9030e-11, 2.8228e-10, 1.1039e-10, 4.4343e-11, 6.8894e-11,\n 1.8676e-12, 3.1666e-10, 1.4415e-11, 2.8238e-10, 1.7259e-12, 9.4751e-12,\n 5.5607e-12, 3.8130e-10, 7.9855e-10, 6.7951e-11, 1.3850e-09, 1.4949e-12,\n 1.2297e-10, 1.3340e-11, 7.6049e-12, 2.7298e-12, 1.0013e-09, 2.6467e-12,\n 5.2326e-10, 1.9843e-10, 2.6028e-10, 7.7686e-10, 5.5425e-11, 6.8021e-12,\n 2.2589e-11, 2.6868e-12, 5.3827e-12, 4.7124e-10, 4.9662e-12, 5.4407e-10,\n 2.4774e-10, 2.3456e-11, 7.9926e-11, 7.2720e-11, 3.4175e-12, 4.2239e-10,\n 2.3701e-12, 2.8438e-11, 3.4689e-10, 1.2179e-10, 4.3229e-10, 2.1876e-12,\n 7.3004e-11, 7.1410e-13, 2.7837e-11, 5.5355e-12, 2.6345e-10, 1.0115e-09,\n 1.0514e-12, 6.7229e-11, 4.1085e-12, 9.3893e-11, 2.5207e-11, 7.0577e-11,\n 4.4840e-12, 1.4430e-11, 1.0692e-10, 3.1340e-11, 1.1958e-12, 6.7146e-12,\n 7.3608e-10, 4.7472e-13, 1.2830e-09, 6.2913e-10, 1.2130e-09, 5.9586e-11,\n 2.5985e-13, 5.6699e-12, 1.0525e-11, 1.3589e-09, 7.0020e-12, 2.8099e-12,\n 2.8654e-10, 3.2757e-12, 3.0762e-11, 1.1776e-11, 2.2769e-12, 5.2457e-10,\n 1.0055e-10, 2.7578e-13, 6.9741e-11, 5.8054e-11, 1.4407e-11, 1.9833e-10,\n 1.6317e-10, 2.2048e-10, 5.4972e-10, 6.8240e-10, 4.4118e-11, 1.9522e-12,\n 1.9027e-11, 4.7940e-10, 2.6518e-10, 7.0867e-11, 1.2362e-11, 3.5446e-10,\n 8.4608e-13, 6.6161e-11, 2.6912e-12, 8.9734e-12, 1.2084e-09, 1.6990e-09,\n 6.7410e-13, 1.7125e-10, 8.6372e-10, 3.2330e-11, 5.4213e-11, 2.5072e-10,\n 9.4848e-11, 3.6558e-10, 4.3282e-13, 2.3523e-11, 3.2096e-11, 8.8258e-12,\n 1.0088e-09, 1.2201e-12, 3.1402e-10, 1.2435e-11, 1.8214e-11, 8.9162e-10,\n 1.3771e-11, 1.5854e-10, 3.1009e-10, 8.9598e-12, 2.3474e-10, 1.0537e-11,\n 1.6508e-10, 5.8628e-10, 1.9982e-11, 5.0676e-10, 1.7533e-13, 4.6157e-11,\n 9.3101e-12, 1.9016e-11, 3.0313e-12, 6.2143e-12, 3.9252e-13, 8.3219e-11,\n 9.8479e-10, 7.0785e-12, 5.9238e-11, 4.5152e-12, 6.9953e-11, 8.5659e-11,\n 8.0376e-10, 7.3273e-11, 7.9884e-12, 2.8326e-11, 1.1622e-13, 3.2944e-10,\n 9.0419e-10, 1.5343e-09, 3.6224e-13, 4.5308e-10, 4.7567e-10, 9.4157e-12,\n 8.0485e-10, 2.0825e-10, 2.6586e-09, 2.2734e-13, 5.7593e-12, 2.5078e-09,\n 2.2229e-10, 1.0952e-12, 9.2973e-13, 3.2551e-10, 7.9645e-10, 8.0015e-12,\n 6.7580e-10, 2.4730e-11, 2.4660e-12, 1.0289e-10, 1.1641e-11, 1.1743e-12,\n 2.5642e-11, 9.3702e-11, 3.9553e-10, 4.4694e-13, 4.8805e-11, 1.9679e-10,\n 1.8919e-11, 1.7101e-11, 1.1312e-10, 2.9477e-12, 1.5953e-10, 8.1980e-11,\n 1.0018e-09, 6.2516e-12, 1.8472e-10, 5.7854e-10, 4.7656e-13, 5.7349e-11,\n 3.0724e-12, 1.2728e-11, 2.6216e-09, 5.7394e-13, 4.6393e-13, 1.8277e-13,\n 3.5868e-13, 2.2506e-11, 1.1656e-10, 4.1942e-10, 2.6504e-10, 2.5498e-12,\n 1.8441e-12, 1.2065e-11, 3.6503e-12, 3.3836e-10, 1.2569e-10, 2.0636e-11,\n 5.1622e-13, 1.7697e-10, 1.3861e-10, 1.1044e-11, 1.6950e-10, 1.3448e-11,\n 8.8611e-10, 2.6977e-11, 1.0269e-10, 6.8395e-13, 4.8987e-10, 2.4504e-09,\n 2.3999e-11, 5.9758e-11, 3.2785e-10, 1.4378e-10, 3.5337e-10, 7.1722e-10,\n 2.2702e-12, 8.2745e-12, 4.2952e-10, 2.1896e-11, 2.2018e-11, 4.8057e-10,\n 1.1179e-11, 1.6175e-09, 2.9027e-13, 6.5497e-12, 5.7076e-13, 1.6399e-10,\n 2.6689e-10, 4.0396e-12, 2.7168e-10, 1.0100e-11, 7.8774e-12, 2.8825e-12,\n 1.4022e-11, 2.9076e-12, 4.9009e-10, 9.7550e-10, 5.4917e-13, 1.7011e-10,\n 8.9036e-11, 1.4168e-12, 1.2823e-11, 1.0304e-10, 4.7108e-11, 5.9603e-11,\n 1.1701e-12, 9.8374e-10, 5.1228e-12, 2.2540e-10, 6.6955e-13, 9.3847e-12,\n 1.2252e-09, 2.2348e-12, 2.8883e-10, 4.9138e-11, 5.0300e-12, 1.3046e-09,\n 3.1783e-12, 2.1146e-10, 6.8035e-13, 2.5578e-12, 1.3976e-10, 6.3387e-11,\n 3.3282e-10, 1.1571e-12, 1.5139e-11, 2.7638e-11, 2.6788e-11, 1.1610e-11,\n 4.7520e-11, 3.8667e-13, 1.8273e-10, 7.6657e-12, 6.4943e-10, 2.5014e-11,\n 1.0165e-09, 1.9036e-10, 1.7375e-09, 7.9148e-11, 8.6311e-12, 3.5330e-12,\n 1.8204e-10, 8.1375e-10, 5.3496e-10, 5.4245e-12, 8.5253e-11, 4.4353e-11,\n 1.9067e-11, 6.5128e-10, 3.6312e-12, 6.4968e-11, 1.2534e-11, 2.1242e-12,\n 2.2097e-11, 1.1055e-12, 1.7758e-10, 7.2238e-12, 1.8953e-11, 1.7878e-11,\n 5.7644e-12, 3.3058e-10, 1.1158e-11, 1.7513e-12, 4.2498e-10, 1.1127e-10,\n 5.0817e-12, 9.0818e-12, 6.4780e-11, 2.7763e-12, 6.1544e-11, 1.2205e-12,\n 5.1670e-12, 4.4336e-10, 2.6160e-10, 2.7606e-11, 5.5589e-10, 9.7010e-11,\n 2.6472e-12, 2.0749e-11, 6.1797e-10, 1.5266e-10, 2.3525e-10, 4.8641e-12,\n 4.7292e-13, 1.3802e-10, 2.8557e-12, 3.3495e-10, 8.1560e-10, 2.4935e-11,\n 1.5039e-10, 7.3723e-11, 2.9225e-11, 4.1783e-10, 5.1504e-12, 3.2577e-12,\n 2.1476e-10, 4.9921e-10, 3.0931e-09, 5.1910e-10, 8.0165e-12, 2.1583e-10,\n 8.5503e-10, 1.3494e-09, 3.1693e-12, 2.8035e-10, 1.0671e-11, 4.2392e-11,\n 3.7697e-11, 2.9115e-12, 6.6065e-13, 1.2496e-11, 2.5094e-12, 3.8088e-12,\n 2.1765e-11, 2.0244e-10, 1.5053e-10, 8.0459e-11, 9.1921e-11, 1.2867e-11,\n 2.6634e-10, 3.0743e-11, 5.2110e-10, 2.1206e-12, 1.6424e-10, 2.1368e-12,\n 4.6627e-12, 7.1383e-12, 2.1559e-11, 3.6464e-10, 2.6579e-13, 9.0389e-10,\n 1.0800e-10, 1.6334e-10, 6.0557e-12, 4.4629e-12, 2.8129e-10, 5.0577e-11,\n 2.3545e-12, 3.9239e-11, 1.9239e-12, 9.5247e-10, 4.0686e-10, 6.5158e-12,\n 3.7337e-10, 2.5093e-10, 4.8271e-10, 3.8332e-12, 1.9157e-10, 1.0003e-10,\n 7.4330e-13, 2.5377e-12, 3.2921e-11, 1.0945e-10, 1.8873e-12, 1.5798e-10,\n 1.4367e-10, 2.1487e-13, 1.7235e-12, 8.7465e-12, 8.3762e-10, 2.5565e-12,\n 2.7467e-12, 6.1517e-11, 2.4777e-10, 1.4356e-11, 3.9003e-11, 8.0890e-14,\n 2.6513e-10, 8.0381e-13, 1.6168e-10, 9.1356e-12, 1.5628e-10, 1.7504e-12,\n 8.1587e-10, 1.0931e-10, 1.0113e-10, 8.4942e-12, 8.9801e-13, 2.3522e-12,\n 3.1900e-12, 2.2467e-11, 8.6824e-11, 5.3028e-13, 3.7559e-11, 3.4805e-10,\n 4.3442e-10, 3.0905e-12, 2.9125e-13, 8.3341e-10, 1.5348e-10, 1.8987e-10,\n 2.0926e-10, 2.1714e-11, 1.0418e-10, 2.7746e-10, 1.8151e-11, 1.1875e-09,\n 4.9628e-10, 6.2992e-11, 1.5179e-12, 1.8091e-09, 4.7574e-11, 1.6174e-10,\n 3.3896e-10, 7.3133e-12, 7.1273e-11, 6.4385e-12, 5.9466e-12, 2.0169e-11,\n 3.7681e-11, 8.0114e-10, 2.7602e-10, 1.0565e-10, 2.3342e-09, 2.3061e-12,\n 8.7436e-11, 1.5154e-12, 4.6751e-10, 8.8119e-12, 3.0707e-10, 6.7804e-11,\n 2.2894e-10, 1.4201e-12, 5.1656e-11, 4.1341e-13, 3.9822e-12, 2.7522e-11,\n 2.7681e-11, 2.9257e-10, 2.4567e-10, 1.4193e-09, 3.3340e-10, 1.0769e-11,\n 2.0993e-12, 1.1534e-11, 5.6981e-11, 5.7379e-10, 5.0605e-12, 1.1764e-11,\n 1.6307e-11, 2.1813e-10, 2.3016e-10, 5.9098e-11, 1.9685e-11, 3.0739e-11,\n 1.1002e-12, 5.8031e-10, 3.8192e-10, 1.9506e-12, 1.7754e-09, 1.3644e-11,\n 7.1499e-11, 9.0622e-10, 1.5008e-11, 8.5269e-10, 4.8260e-13, 2.2436e-11,\n 1.4192e-12, 1.2636e-10, 6.2236e-12, 2.6412e-11, 6.7851e-10, 1.7428e-12,\n 7.5336e-10, 1.0141e-11, 2.2311e-10, 9.6826e-10, 2.3539e-12, 1.7053e-09,\n 7.4534e-11, 7.2265e-13, 2.9191e-12, 5.8810e-11, 2.4054e-10, 4.0980e-10,\n 2.9545e-10, 1.0465e-12, 7.3773e-11, 4.8786e-10, 2.7782e-10, 2.7506e-12,\n 5.8271e-10, 3.9347e-11, 9.9336e-12, 6.3282e-11, 3.3045e-11, 2.5801e-11,\n 1.3555e-10, 2.6287e-10, 2.9408e-12, 9.1027e-11, 2.5950e-11, 1.6535e-12,\n 1.1969e-12, 2.5431e-11, 3.6497e-11, 3.1251e-12, 6.0851e-13, 1.9230e-11,\n 6.2736e-12, 1.4204e-12, 1.5855e-11, 7.0656e-13, 1.1694e-10, 4.0936e-12,\n 1.9363e-11, 9.5683e-10, 2.7101e-11, 1.3501e-09, 1.1990e-12, 2.3416e-12,\n 1.8726e-10, 4.4858e-12, 4.1889e-11, 1.9485e-12, 2.8576e-10, 4.6203e-11,\n 6.4169e-11, 4.9119e-12, 3.5832e-12, 3.4992e-10, 3.5949e-11, 1.5472e-11,\n 5.1370e-10, 1.9677e-10, 6.1695e-10, 1.5774e-10, 1.0296e-11, 1.3622e-11,\n 1.2912e-11, 1.1296e-11, 1.7215e-11, 1.3377e-12, 9.6931e-12, 7.5385e-13,\n 1.4424e-12, 2.2270e-11, 1.7865e-11, 6.4934e-10], device='cuda:0')" + }, + "46": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 5.4661e-18, 2.3441e-18, -5.6052e-45, ..., 1.3381e-16,\n 1.8478e-15, 5.6052e-45],\n [-2.0191e-16, -2.8359e-16, -5.6052e-45, ..., -6.8791e-15,\n -3.9331e-14, -5.6052e-45],\n [ 6.5952e-17, -4.8872e-17, -5.6052e-45, ..., 1.3809e-15,\n 1.1449e-14, 5.6052e-45],\n ...,\n [ 2.8568e-17, -2.6831e-17, 5.6052e-45, ..., 1.0645e-16,\n 1.8457e-15, 5.6052e-45],\n [-1.3162e-17, -1.2323e-16, 5.6052e-45, ..., -2.1536e-15,\n 6.6006e-15, 5.6052e-45],\n [-7.7819e-17, -3.9747e-17, 5.6052e-45, ..., -2.9614e-15,\n -1.0643e-14, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[4.4484e-12, 2.2610e-12, 5.0818e-16, ..., 1.1798e-11, 1.7117e-12,\n 5.1897e-14],\n [2.1607e-12, 1.1655e-12, 5.5734e-16, ..., 2.6633e-12, 7.2878e-13,\n 3.1354e-13],\n [3.3931e-13, 2.7948e-13, 2.0065e-17, ..., 6.7553e-13, 7.2363e-14,\n 1.0510e-13],\n ...,\n [1.2959e-11, 9.8744e-12, 2.4799e-16, ..., 1.8841e-11, 3.1630e-12,\n 3.7526e-14],\n [3.2229e-12, 4.1241e-12, 4.9051e-15, ..., 7.5433e-12, 1.0941e-12,\n 5.3875e-14],\n [6.2331e-11, 7.8627e-11, 3.5974e-15, ..., 1.1951e-10, 2.5147e-11,\n 1.7860e-13]], device='cuda:0')" + }, + "47": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 3.6026e-13, -3.6488e-12, 1.2418e-12, 1.5658e-13, -3.5253e-14,\n -7.6934e-13, 2.1897e-13, 1.9373e-13, 5.9184e-13, 1.3465e-13,\n -1.0776e-14, 1.4326e-13, -1.2350e-12, -9.0964e-14, -4.4411e-13,\n 5.2803e-13, -8.0895e-13, 8.4991e-13, 8.2289e-13, 2.2723e-13,\n 2.1234e-13, 6.1944e-14, 2.7945e-13, 7.6272e-13, -5.6332e-14,\n -2.9207e-13, 1.6047e-13, 1.1754e-12, -5.6675e-14, -6.3744e-13,\n 2.6543e-13, 1.3754e-13, 7.4465e-13, -8.1775e-13, 5.8915e-14,\n -6.6158e-14, -3.1865e-14, -1.4243e-13, -1.7104e-12, 1.7219e-13,\n 2.4137e-13, 6.8133e-14, 7.6863e-14, -6.7122e-13, -8.7681e-13,\n -3.1982e-13, -3.3220e-13, -6.7228e-14, 3.4326e-14, -8.1102e-13,\n -3.7020e-14, 9.9014e-14, 7.9824e-14, 9.9809e-14, 1.0345e-13,\n 5.0985e-14, -5.6444e-13, 8.2112e-13, 4.2051e-15, 3.0290e-14,\n 1.5669e-13, 5.3367e-14, 5.2392e-13, -2.4621e-13, -5.7192e-14,\n 1.2285e-13, -7.5403e-13, 1.0997e-12, -1.8017e-13, 4.9781e-14,\n -5.3022e-13, -2.4032e-13, -3.7232e-15, -1.4494e-15, -3.4739e-14,\n 3.7236e-14, -1.3907e-13, 6.1214e-13, -7.0889e-13, -1.4096e-13,\n 1.1736e-12, -1.5225e-13, 9.9857e-13, 3.8415e-14, -1.1858e-13,\n -1.0005e-12, -2.9167e-13, -3.0032e-13, -3.7479e-13, 2.7502e-13,\n 1.8006e-13, 4.4235e-13, -6.1564e-13, 1.1244e-13, 7.9983e-14,\n 9.1723e-14, -7.0102e-13, -3.5710e-13, 1.0573e-13, 6.2470e-13,\n -6.9218e-13, 1.7724e-13, -5.1559e-13, -3.1432e-14, 4.6238e-13,\n -4.1260e-14, -1.7707e-13, 4.9808e-13, -7.9888e-14, -1.9326e-13,\n 1.1347e-13, -1.8198e-13, -7.2418e-14, -1.0468e-12, 1.3442e-12,\n 4.4775e-13, 4.2526e-13, 1.4891e-12, -8.4804e-13, 1.9946e-13,\n -4.6260e-13, 6.2529e-13, 4.5369e-13, 2.6524e-13, 1.8043e-13,\n 2.4721e-14, -2.5340e-12, -2.2888e-14, 5.4640e-13, 9.3125e-13,\n 3.6077e-13, 4.1548e-14, 4.1772e-13, 7.9806e-14, 2.0193e-13,\n -9.8517e-13, 9.5934e-14, -8.7396e-14, 5.7879e-14, -5.3242e-14,\n -2.2073e-13, -3.9762e-13, 6.3564e-13, 2.9110e-13, 6.0535e-13,\n 1.3326e-13, 5.2104e-13, -2.6117e-14, 1.4326e-13, -6.2626e-14,\n 1.5696e-13, -2.4305e-13, 8.0119e-14, -2.0378e-13, 1.7337e-13,\n 1.6714e-13, 4.5871e-13, -8.3802e-13, -5.8631e-14, 1.2137e-13,\n 6.5603e-13, -3.0174e-13, -8.8483e-13, 4.7978e-14, -3.5731e-13,\n 5.7760e-13, -1.2630e-13, 1.5047e-13, -1.7859e-13, 1.0089e-12,\n 8.1326e-13, 3.2740e-14, -1.3550e-15, -8.5673e-13, -9.6078e-13,\n 7.7172e-13, -3.9958e-14, 2.6259e-13, -3.3449e-13, -1.2551e-13,\n 2.8507e-13, 7.7384e-13, -1.5037e-12, 1.7858e-13, -1.0097e-12,\n 1.5439e-12, -7.1741e-13, -6.1739e-14, -5.7143e-13, 8.1008e-13,\n 9.4175e-13, -8.2766e-14, 7.5286e-14, 1.4323e-13, -4.5414e-14,\n -2.2337e-13, 5.1606e-13, 6.8246e-14, 1.0583e-13, -9.0979e-13,\n 3.0343e-13, 8.1612e-14, -9.8366e-14, -3.5506e-13, 5.4517e-13,\n 9.2126e-13, -1.5493e-13, -1.2751e-12, -3.1002e-14, -2.1357e-14,\n -2.8275e-14, 2.7890e-13, 2.3034e-13, 2.2646e-13, 5.0261e-13,\n 2.2321e-12, 9.3081e-14, 3.1304e-13, 2.7930e-13, 1.5557e-12,\n 7.4652e-15, -1.5271e-13, -8.9501e-14, -2.4991e-13, -1.1095e-13,\n -2.8060e-13, 2.8646e-13, -5.7339e-13, -1.3377e-12, 1.2717e-13,\n 8.8247e-14, 8.4129e-14, 2.3755e-13, -7.4689e-13, 4.6669e-14,\n -9.6170e-14, -2.5403e-13, -2.8380e-14, 8.7945e-14, 5.7186e-13,\n -1.7320e-14, -7.7113e-15, 5.0671e-14, -1.0920e-13, 3.0151e-13,\n 2.0442e-12, -7.8956e-14, -1.9687e-13, 1.4017e-13, -4.2543e-13,\n 6.3858e-13, 5.4302e-13, 1.5065e-12, 6.5618e-14, 9.5900e-14,\n -6.5896e-14, -6.9053e-13, -1.9206e-13, -9.4944e-13, 4.7351e-13,\n -3.9492e-13, 9.1625e-15, -1.4427e-13, -2.9835e-14, 1.1237e-12,\n -1.3595e-13, -1.3731e-13, 4.5447e-13, 7.5104e-13, -1.5475e-12,\n 3.7993e-14, 4.5965e-14, 1.4984e-13, -1.2513e-12, 4.3220e-13,\n -5.9125e-13, -1.2511e-12, 1.9656e-13, 8.2581e-13, -3.6278e-13,\n 4.7925e-13, 1.3288e-12, -3.3989e-13, -2.1355e-12, 1.8558e-13,\n -1.2637e-12, 4.4144e-14, -4.3079e-13, -1.1136e-13, 6.2241e-13,\n -3.7196e-14, -8.3806e-14, -1.3529e-12, 4.0461e-14, 1.1847e-13,\n -1.3653e-12, -3.6721e-13, 1.4943e-13, 2.0878e-15, -1.6754e-13,\n 8.0758e-13, -5.7375e-13, 4.1545e-13, 6.8977e-14, -7.0497e-13,\n 2.8836e-13, -1.9474e-13, -2.4141e-13, -3.7299e-13, 7.6475e-14,\n -1.5954e-14, 1.7378e-13, 1.5345e-13, 3.3157e-13, -7.7582e-14,\n -3.4303e-13, -1.1759e-12, 2.0249e-13, -9.7454e-13, -1.7653e-13,\n 5.7790e-14, -1.5857e-13, 3.9718e-14, 5.6071e-14, 2.0660e-13,\n -4.1392e-14, -7.7401e-14, 6.9627e-13, 1.0966e-13, -6.1390e-13,\n -1.5611e-13, 1.7081e-13, -9.0269e-14, -2.8932e-13, 3.0797e-14,\n 4.5003e-13, 3.4042e-14, 1.9692e-13, -2.0928e-13, 4.3218e-13,\n -1.1496e-13, 7.0837e-13, -9.1386e-13, 1.4621e-13, -1.8995e-12,\n 3.7107e-13, 7.6061e-13, 1.8698e-14, -2.5295e-13, -1.3805e-13,\n 2.7566e-13, -5.0508e-14, -8.8714e-14, 1.6393e-13, 3.5853e-13,\n -9.6665e-14, -4.5640e-13, 5.5658e-13, -1.8216e-13, -1.1941e-13,\n 1.1289e-13, -5.3142e-13, 4.5137e-13, -3.4286e-13, 3.7263e-14,\n -2.3082e-14, 7.8259e-13, 6.1311e-13, 5.7403e-14, -9.7466e-13,\n 7.1851e-13, 1.2583e-13, 2.1225e-13, 3.7501e-13, -1.6441e-14,\n -1.8045e-14, 1.6357e-13, -3.2574e-13, 2.5176e-13, 2.3624e-13,\n -1.0246e-12, 1.7167e-12, 1.5590e-12, 1.1833e-12, -2.1547e-13,\n 1.9291e-14, -9.4647e-13, 1.3746e-12, 7.3663e-14, 6.8801e-14,\n 8.7349e-14, 1.5806e-13, 3.9002e-13, 1.3064e-12, -6.3129e-13,\n -4.6406e-14, 1.0549e-13, -1.9620e-12, -1.2934e-12, -1.4575e-13,\n -1.0047e-12, 6.4163e-14, -6.3114e-13, -3.3288e-14, -4.3828e-13,\n -3.9405e-15, 7.0946e-13, -1.0716e-13, -1.5011e-13, -3.4219e-13,\n -9.2452e-13, -2.8440e-13, -1.6264e-12, -1.7563e-14, 4.5752e-13,\n -7.7273e-13, 1.1900e-13, 1.0947e-12, -2.7366e-14, -3.4791e-13,\n 1.3213e-14, 1.3962e-13, -6.1909e-13, -7.4673e-13, 1.3577e-13,\n -1.1631e-13, -2.3935e-13, 6.0428e-13, -1.7823e-13, 5.9341e-14,\n 1.2907e-13, 9.2645e-14, 2.3315e-13, 2.4960e-13, 2.6723e-13,\n -8.3565e-13, 2.4087e-13, -5.8418e-13, 5.8301e-13, 4.2701e-15,\n -9.9030e-14, 2.2679e-14, 1.4918e-13, -1.1670e-13, 1.3921e-14,\n 3.0289e-13, -8.0066e-13, 1.2452e-13, 3.6667e-14, -6.4832e-14,\n -7.0583e-13, -1.9631e-13, -2.1839e-13, -2.7034e-13, 2.8563e-12,\n 1.6559e-13, 8.1791e-14, 4.6251e-14, 3.0831e-13, -1.1757e-13,\n -2.4280e-14, -1.5266e-15, -1.1916e-14, -1.0909e-12, -4.2019e-14,\n 1.4434e-12, 3.9917e-13, 1.3876e-13, 6.6761e-13, 8.9968e-13,\n 2.8943e-13, 9.4802e-14, -4.1102e-13, 9.9292e-13, 9.3248e-13,\n -5.6198e-13, 3.2685e-13, 1.4640e-13, 1.2194e-13, 1.9939e-13,\n 9.9177e-15, 3.3750e-13, 1.1153e-12, -2.9685e-13, 1.4076e-13,\n -6.3302e-13, 3.4131e-13, -1.0488e-13, -8.4760e-14, -2.2593e-12,\n -2.4834e-13, -5.7599e-13, -1.3818e-14, 5.6933e-14, -2.5848e-13,\n 4.8730e-14, -1.0891e-13, -3.5183e-15, -1.4837e-13, 6.2100e-13,\n 6.0744e-14, -7.1606e-14, -1.7294e-13, 1.2004e-14, 4.7770e-13,\n -2.5661e-14, -1.5915e-13, -4.9837e-14, -1.1975e-14, -3.3964e-13,\n -4.3339e-13, 1.8874e-13, 1.9497e-13, -2.0392e-13, 8.9633e-14,\n -1.3392e-13, 2.0087e-13, -1.5039e-12, 4.0116e-13, 5.3811e-14,\n -6.8341e-13, -8.0417e-14, 1.8437e-13, 6.6825e-13, -4.6117e-14,\n -1.4447e-13, -2.3135e-13, 5.7323e-13, -5.7239e-13, -6.0326e-13,\n -6.0030e-13, 1.4963e-12, 2.1602e-13, -5.4773e-15, -1.0912e-13,\n -2.3746e-15, 4.4034e-13, 2.1357e-12, 9.3117e-14, -3.2400e-13,\n 3.1526e-14, -2.0034e-13, 1.1057e-12, 2.0558e-13, -3.3322e-13,\n -1.5614e-12, 1.2694e-12, 3.1738e-13, -1.9953e-12, 2.9756e-13,\n -1.0622e-12, -2.9807e-13, 1.0970e-12, -1.7945e-13, 7.6253e-13,\n 7.7436e-16, 1.8640e-13, 1.0152e-13, 5.3115e-14, 4.9035e-14,\n 8.3723e-13, 1.8682e-13, -1.7537e-13, 2.0002e-12, 5.1370e-13,\n 9.2664e-15, 7.4957e-13, -1.3365e-12, 1.9838e-12, -6.9193e-13,\n 4.2185e-13, -4.0635e-14, 2.8692e-14, -1.1953e-12, -7.7819e-13,\n -3.5794e-13, -2.7087e-13, -1.8608e-15, 2.4696e-14, 3.6300e-14,\n -1.8485e-12, 6.6979e-14, -1.1493e-12, -1.3181e-13, -9.1339e-14,\n -7.5388e-13, -4.3732e-13, -5.0460e-14, 5.9682e-14, -1.6085e-14,\n 1.1520e-12, 4.4969e-14, 7.2328e-14, 1.2630e-12, 6.1108e-13,\n -7.1048e-13, 4.3871e-13, -7.2432e-13, -6.3744e-13, 1.6200e-12,\n -4.7592e-13, -2.6893e-14, 2.0866e-14, 1.7237e-12, 3.0112e-13,\n -3.9876e-13, -1.1099e-12, 1.0202e-12, 4.0012e-13, -9.7866e-13,\n 1.3845e-12, 1.0425e-14, -4.7035e-14, -2.7939e-14, -3.0146e-14,\n -3.8590e-13, -9.6518e-13, 1.4257e-15, 1.3619e-13, 4.5752e-14,\n -8.6764e-14, -9.2097e-13, -4.9811e-13, -2.3879e-13, 3.7624e-13,\n -2.3570e-13, 4.0499e-14, -3.7834e-13, -3.9572e-14, -8.5920e-13,\n 1.4808e-12, 4.6569e-13, 1.1428e-13, 2.7431e-14, -3.4011e-14,\n 1.4473e-13, -3.5552e-13, 5.4450e-14, -2.4936e-13, -9.8715e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.3826e-07, 8.7096e-08, 7.3021e-09, 7.9271e-09, 7.8034e-09, 3.2064e-08,\n 1.3202e-06, 2.6677e-06, 3.1024e-08, 1.0091e-08, 4.7359e-07, 6.2581e-07,\n 1.8258e-06, 2.6843e-07, 1.0544e-07, 8.0022e-08, 1.4910e-06, 1.6606e-06,\n 4.3992e-07, 6.2037e-09, 1.0647e-07, 8.8854e-08, 1.1760e-07, 7.9707e-08,\n 6.5576e-08, 1.4609e-08, 3.4346e-08, 5.8279e-07, 3.0158e-08, 5.7432e-07,\n 2.3957e-06, 4.7206e-08, 1.2632e-06, 1.1844e-08, 5.6174e-07, 6.5388e-07,\n 6.7665e-07, 2.2657e-09, 1.2145e-07, 5.3502e-09, 3.3106e-08, 1.6317e-08,\n 1.4824e-06, 8.3743e-08, 1.7796e-07, 3.3382e-07, 1.7164e-08, 5.4963e-09,\n 4.1263e-09, 9.6931e-07, 4.7023e-09, 1.2880e-08, 5.6472e-09, 5.4684e-09,\n 1.6839e-08, 9.1638e-08, 6.8555e-08, 1.0374e-06, 2.2654e-06, 7.1931e-07,\n 3.7816e-09, 2.4814e-07, 2.0043e-06, 7.0002e-09, 1.6864e-08, 3.6061e-08,\n 2.9104e-06, 2.6621e-07, 7.3722e-10, 1.2958e-07, 1.0670e-06, 5.3181e-09,\n 2.7601e-08, 4.8402e-09, 1.9250e-08, 2.0527e-09, 7.5219e-08, 2.7622e-06,\n 1.8210e-08, 3.1168e-06, 3.4595e-07, 2.8243e-09, 5.2544e-09, 1.8847e-08,\n 4.2050e-08, 5.5855e-09, 3.6929e-08, 5.5120e-08, 1.2907e-06, 9.8560e-07,\n 3.6300e-08, 9.8406e-09, 1.5635e-06, 1.6001e-06, 3.9453e-09, 7.1610e-09,\n 2.0248e-07, 4.8768e-06, 1.4430e-08, 7.0823e-08, 7.3866e-08, 1.9976e-07,\n 5.6086e-07, 1.5681e-07, 9.1936e-07, 6.6771e-07, 1.2917e-08, 2.1761e-08,\n 9.5560e-07, 5.4318e-08, 1.0326e-08, 7.1148e-08, 2.9872e-06, 1.9073e-07,\n 1.7853e-07, 3.6536e-07, 8.3363e-08, 7.1059e-06, 4.9322e-08, 2.7090e-09,\n 1.6687e-06, 1.3290e-06, 1.5459e-07, 2.2114e-07, 1.2949e-06, 1.0381e-08,\n 4.8661e-08, 7.0751e-09, 3.0868e-07, 4.7328e-07, 2.5702e-07, 2.0753e-08,\n 2.0368e-06, 2.6959e-08, 3.1567e-06, 2.4003e-06, 6.3343e-08, 1.9727e-09,\n 1.2225e-06, 4.1817e-08, 5.1105e-09, 8.5660e-09, 6.9981e-07, 1.5592e-06,\n 3.0636e-08, 8.1223e-09, 4.2621e-08, 5.5837e-06, 8.6061e-08, 1.0560e-06,\n 3.9884e-09, 9.9810e-09, 1.0235e-06, 8.0637e-08, 7.0577e-09, 8.3320e-07,\n 9.3817e-09, 7.5120e-07, 3.0055e-09, 6.0830e-09, 2.6698e-06, 3.4332e-09,\n 1.4732e-06, 8.0954e-09, 3.8198e-07, 6.3790e-09, 2.1196e-08, 4.1648e-07,\n 7.3098e-09, 9.2466e-08, 1.1136e-07, 3.1240e-07, 1.8367e-07, 3.6267e-08,\n 2.3969e-07, 1.0280e-06, 1.2025e-08, 1.9987e-06, 4.2273e-09, 1.8548e-07,\n 1.9747e-07, 4.3688e-08, 7.1519e-07, 3.8682e-07, 5.6001e-08, 2.7208e-07,\n 2.6799e-06, 1.4230e-07, 1.9848e-06, 1.8071e-07, 4.3468e-07, 4.1072e-07,\n 3.2254e-08, 4.3007e-09, 8.5460e-08, 1.6786e-06, 1.4278e-08, 4.0060e-08,\n 1.2332e-07, 8.6025e-07, 1.2593e-08, 1.2438e-08, 3.8987e-06, 6.1403e-08,\n 1.8970e-09, 1.7622e-06, 9.8375e-08, 6.2872e-08, 5.9761e-08, 1.3306e-07,\n 5.6371e-09, 1.2565e-06, 7.6872e-08, 1.2815e-06, 1.7170e-06, 1.5271e-07,\n 9.6606e-07, 2.7690e-08, 9.5338e-08, 3.9734e-06, 7.9146e-09, 2.2835e-08,\n 2.1864e-06, 4.4466e-07, 2.8013e-07, 8.4524e-08, 2.0210e-06, 1.3456e-08,\n 7.5723e-07, 1.2445e-08, 1.5179e-08, 9.7228e-09, 1.1312e-06, 2.9540e-06,\n 1.1973e-07, 9.8846e-08, 5.6723e-07, 8.5537e-08, 1.1906e-07, 4.8670e-09,\n 1.0161e-08, 5.3078e-09, 1.0285e-07, 1.0950e-07, 1.3053e-07, 3.8287e-09,\n 3.5087e-07, 3.2389e-08, 1.5458e-08, 5.7148e-07, 1.3526e-06, 1.6892e-06,\n 1.7613e-08, 7.5222e-09, 1.1817e-07, 3.9657e-06, 4.7393e-07, 1.3720e-08,\n 7.9466e-07, 1.9680e-06, 6.0030e-09, 2.4607e-08, 1.7416e-07, 2.5929e-07,\n 2.3390e-06, 3.7092e-06, 1.3933e-08, 6.3838e-07, 1.6914e-06, 9.6545e-08,\n 1.6207e-08, 1.2731e-08, 1.0710e-06, 4.2920e-07, 9.6022e-07, 4.8219e-07,\n 1.8635e-07, 3.6075e-09, 2.7543e-06, 6.4735e-07, 7.3943e-08, 1.3985e-06,\n 1.3849e-08, 2.5985e-06, 1.0526e-07, 1.0298e-07, 3.8420e-09, 1.1589e-06,\n 8.3844e-09, 5.8682e-09, 5.4780e-08, 1.2473e-08, 1.4246e-06, 1.6773e-08,\n 2.7493e-08, 2.5155e-06, 4.0278e-06, 3.0320e-09, 4.6685e-09, 4.9952e-08,\n 2.0554e-07, 3.5650e-07, 3.4193e-06, 1.6943e-07, 1.1785e-06, 2.8756e-07,\n 1.3846e-07, 1.1775e-06, 8.9255e-09, 3.7215e-09, 6.1708e-09, 1.9002e-08,\n 2.3601e-08, 2.1057e-06, 6.1122e-08, 7.6953e-09, 1.2333e-08, 1.2726e-07,\n 1.4706e-06, 4.5774e-08, 1.7108e-08, 5.2412e-07, 2.3421e-08, 8.0104e-09,\n 4.9159e-08, 2.7237e-09, 4.9785e-09, 1.5951e-08, 4.6411e-08, 1.1115e-06,\n 7.4868e-08, 2.8830e-07, 6.3559e-07, 6.2219e-08, 3.4348e-08, 4.0309e-08,\n 1.7519e-08, 5.9439e-08, 7.6414e-08, 3.6710e-06, 5.8492e-07, 5.0258e-08,\n 4.8610e-07, 1.2484e-09, 1.9918e-07, 9.1005e-08, 1.2004e-07, 4.2753e-07,\n 3.1851e-08, 1.0971e-06, 5.8642e-09, 1.8045e-08, 1.2210e-08, 2.9530e-07,\n 7.2766e-09, 3.7731e-09, 4.7998e-07, 3.0452e-07, 6.7212e-08, 4.5391e-08,\n 2.5994e-06, 9.0369e-07, 3.0436e-08, 8.7052e-08, 5.5677e-07, 2.8667e-08,\n 6.7389e-08, 9.7594e-07, 4.3543e-08, 2.0201e-06, 8.7973e-07, 3.6093e-09,\n 1.8905e-07, 3.0242e-06, 1.0794e-08, 5.4164e-09, 5.5871e-08, 1.7196e-07,\n 3.1988e-06, 2.1666e-07, 1.3174e-07, 9.0145e-08, 1.3346e-08, 1.4284e-07,\n 1.0709e-07, 7.3429e-08, 1.3328e-06, 2.3545e-07, 1.4462e-08, 8.7036e-09,\n 1.2625e-07, 7.9886e-09, 3.7685e-06, 7.3541e-07, 8.2215e-07, 3.9764e-09,\n 1.8254e-09, 1.1488e-06, 7.4820e-06, 1.1549e-06, 5.0589e-07, 1.4450e-06,\n 3.1866e-06, 1.6889e-07, 1.2706e-08, 3.1033e-07, 1.0706e-08, 5.5489e-09,\n 3.8732e-08, 3.6015e-07, 1.4325e-08, 2.7821e-07, 1.6139e-07, 8.1631e-09,\n 4.9686e-09, 1.3122e-07, 2.7718e-08, 8.4995e-07, 5.0466e-07, 5.0253e-08,\n 1.1056e-08, 1.0237e-06, 1.9177e-06, 1.9752e-07, 2.2295e-06, 3.4443e-08,\n 4.8120e-08, 1.6843e-06, 1.1482e-08, 5.4905e-07, 2.8843e-07, 1.4444e-06,\n 2.8206e-09, 1.0711e-06, 8.8799e-09, 1.7597e-07, 5.2144e-07, 2.6109e-08,\n 2.0486e-07, 5.3936e-08, 8.2851e-08, 1.9519e-08, 7.4462e-09, 2.9238e-06,\n 1.4116e-08, 2.6737e-06, 3.6528e-06, 2.4509e-07, 5.1219e-09, 9.4816e-09,\n 2.3491e-08, 9.3223e-07, 1.2965e-07, 2.3073e-09, 8.3464e-08, 8.6149e-08,\n 5.2553e-09, 4.3061e-09, 2.3747e-07, 1.5926e-07, 2.6063e-08, 9.4368e-09,\n 1.6456e-08, 7.1732e-08, 3.1653e-09, 1.1225e-07, 5.2332e-07, 4.1497e-09,\n 2.2451e-07, 5.6634e-07, 4.7950e-08, 4.4637e-08, 7.0286e-09, 5.6267e-07,\n 4.4842e-07, 8.1523e-07, 3.8796e-08, 7.6668e-08, 4.6475e-08, 3.6023e-07,\n 2.4921e-07, 1.6302e-08, 7.6598e-07, 6.3933e-08, 1.7581e-07, 1.2060e-06,\n 9.7800e-07, 2.7869e-09, 2.4909e-08, 3.0913e-06, 4.3167e-08, 3.7003e-06,\n 1.5481e-08, 2.4758e-09, 6.2200e-07, 6.7865e-09, 3.2579e-08, 6.8492e-08,\n 7.2247e-07, 2.8113e-07, 2.0670e-08, 1.8511e-07, 9.1638e-09, 3.4770e-07,\n 2.8114e-08, 3.5391e-08, 5.5857e-09, 9.8604e-08, 5.8556e-09, 3.9206e-07,\n 5.6941e-07, 8.4491e-08, 2.1098e-08, 4.9532e-09, 7.7175e-06, 2.1037e-06,\n 7.7087e-08, 3.1468e-08, 4.3769e-06, 3.4415e-09, 1.6333e-07, 2.4456e-09,\n 3.0078e-07, 3.7484e-07, 1.0842e-07, 5.0609e-07, 7.0938e-08, 1.3912e-07,\n 7.1789e-07, 1.9606e-08, 2.3736e-07, 7.8937e-07, 1.4290e-06, 7.3684e-09,\n 9.4225e-07, 2.2217e-08, 5.4980e-07, 8.6911e-07, 5.7847e-09, 9.7806e-07,\n 2.1739e-08, 2.9132e-07, 2.2646e-06, 6.9009e-07, 3.9368e-08, 7.5390e-08,\n 2.9330e-07, 1.9429e-08, 1.9809e-06, 1.4352e-08, 4.5814e-08, 3.9567e-08,\n 3.0475e-06, 2.7677e-08, 6.3091e-06, 7.5394e-08, 6.1757e-08, 4.7011e-09,\n 7.0184e-09, 3.1816e-07, 2.8374e-08, 2.4531e-06, 6.9412e-07, 7.8728e-09,\n 3.9499e-06, 1.5212e-08, 2.8682e-06, 1.1109e-06, 4.6754e-07, 2.4872e-07,\n 1.4529e-07, 1.9682e-07, 5.3743e-08, 2.1130e-06, 8.8268e-08, 3.0783e-08,\n 8.4051e-07, 3.6825e-09, 4.9591e-09, 4.1553e-06, 2.0382e-07, 5.3501e-09,\n 3.4092e-06, 4.0572e-08, 1.0284e-08, 3.5587e-08, 3.9894e-09, 4.3046e-09,\n 7.2958e-09, 3.6991e-08, 2.2979e-06, 2.1456e-07, 1.3700e-08, 4.8070e-07,\n 3.9046e-08, 1.3155e-06, 2.8575e-08, 2.9441e-08, 2.5918e-08, 2.9133e-07,\n 5.8434e-07, 1.1022e-06, 4.3068e-07, 4.6075e-09, 1.1811e-07, 4.2252e-09,\n 7.9782e-08, 2.9128e-06, 6.3612e-07, 3.5417e-06, 9.3031e-09, 1.6909e-08,\n 1.1224e-07, 1.7546e-08, 3.5693e-08, 3.7802e-07, 2.5484e-07, 1.0338e-08,\n 5.3495e-09, 2.3004e-08, 1.4903e-08, 8.4852e-09, 1.5885e-07, 2.6920e-08,\n 5.7960e-07, 1.8722e-06, 7.0580e-08, 2.1327e-09, 3.4112e-08, 7.8740e-09,\n 2.1981e-06, 4.0323e-08, 1.7420e-08, 9.2817e-09, 3.9157e-08, 1.2654e-07,\n 1.6241e-08, 4.7397e-07, 1.5889e-07, 2.8051e-06], device='cuda:0')" + }, + "48": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.6467e-15, -7.5574e-14, 9.2949e-15, 1.1160e-17, -5.1545e-15,\n -2.5059e-15, 9.0954e-16, -2.2695e-15, 1.5774e-15, -3.2890e-15,\n -1.0034e-15, -7.1882e-16, -3.0249e-14, 8.6771e-16, -4.1664e-15,\n 9.6525e-15, -1.3337e-14, 9.5571e-15, 2.0866e-15, 7.3145e-16,\n -7.1081e-16, -2.4319e-14, 2.2454e-16, 2.9943e-16, -1.7661e-15,\n -2.4500e-14, -1.1501e-15, 9.1043e-15, 1.9339e-15, -2.4307e-14,\n -4.2087e-15, -1.2158e-15, 6.6278e-15, -2.7121e-15, -1.8542e-15,\n -5.2168e-15, -2.2921e-14, 3.6409e-15, -1.1338e-14, 8.8670e-18,\n -5.6520e-16, -3.0965e-16, -8.3201e-15, -3.0137e-14, -7.9883e-14,\n -9.7566e-15, -2.8110e-16, -1.0682e-15, 6.5189e-16, -9.6700e-15,\n 9.7760e-16, -3.9201e-16, 1.5137e-15, -1.2345e-16, -4.0240e-16,\n 2.3327e-15, 3.1489e-15, 4.5578e-15, 1.0711e-15, -6.6235e-16,\n -4.8728e-15, -5.0381e-15, 4.2918e-15, -6.7862e-15, -2.0933e-15,\n -2.0282e-16, 3.5459e-15, 1.7841e-14, -2.4233e-16, -2.7657e-15,\n -3.1160e-16, 6.3870e-15, -5.0217e-15, 1.1690e-15, -2.4408e-16,\n -2.0659e-15, -3.8482e-14, 2.2977e-15, -4.7610e-15, -2.1768e-14,\n 4.1950e-15, -2.4926e-15, -1.9255e-15, -5.5504e-18, -8.0750e-16,\n -1.3327e-14, -4.2844e-16, -5.4733e-14, -2.4602e-14, 4.3825e-15,\n -8.6293e-16, 4.1208e-16, -1.7657e-14, -5.0275e-16, -3.3146e-15,\n -6.6161e-15, -1.0297e-14, -1.8254e-14, -1.0363e-15, 1.1867e-15,\n -4.6417e-14, -2.8775e-15, -4.4903e-15, -1.3551e-14, 4.8736e-15,\n 1.4388e-15, 2.0855e-15, -1.3481e-15, -6.4329e-15, -8.9681e-16,\n -6.9177e-15, 1.8578e-15, -7.7802e-15, -4.8744e-14, 1.2027e-14,\n 9.1449e-16, 3.8038e-15, 9.2719e-15, -1.5474e-14, -9.0818e-15,\n -5.0475e-15, 7.2826e-15, 1.1591e-15, 1.3831e-16, -1.3394e-15,\n 9.7139e-16, -7.7695e-14, 1.3336e-15, 7.2289e-15, 4.0373e-15,\n 2.9771e-15, -2.2425e-15, -8.3504e-16, -2.9555e-16, -2.7403e-15,\n -7.7022e-15, -8.5792e-16, 8.2200e-16, -3.8040e-16, -1.2940e-16,\n -6.1601e-16, -4.6530e-15, 4.8645e-15, 2.0234e-15, 1.4232e-15,\n -2.3830e-16, 5.9787e-15, 2.0858e-15, -1.0674e-15, -2.1725e-14,\n -9.4282e-16, -9.0484e-15, -4.2572e-16, -5.6155e-15, 3.7753e-16,\n 9.5399e-16, -6.4086e-16, -6.7235e-15, 1.0791e-15, -5.4355e-16,\n 4.0051e-15, -5.0994e-15, -1.3089e-14, -3.7337e-17, -7.5936e-15,\n 1.2101e-16, 1.3330e-15, -1.1161e-15, 2.8892e-16, 1.6945e-14,\n 4.8823e-15, 2.5992e-16, 2.0474e-16, -1.3735e-14, -1.7319e-14,\n 7.7421e-15, 2.5703e-16, 2.1656e-15, -2.9340e-16, -3.7011e-15,\n -1.0448e-14, 2.8865e-15, -2.8307e-14, -2.7787e-15, -3.4873e-14,\n 3.0421e-14, -2.6535e-14, 7.3847e-17, -2.9811e-14, 1.7275e-15,\n 7.1926e-15, -2.1528e-14, -4.9318e-16, -4.1190e-16, 2.7781e-15,\n -2.8561e-14, 6.5704e-16, -9.1734e-16, -2.5848e-16, -1.3097e-14,\n -5.8070e-15, -7.9498e-16, 3.4554e-16, -7.8281e-15, 5.3407e-15,\n 6.6477e-15, -9.2702e-15, -1.0065e-13, 3.9922e-16, 5.7035e-16,\n 6.3979e-16, 1.5545e-15, 1.1352e-14, 4.5954e-15, 1.1367e-15,\n 5.2025e-14, -2.1893e-15, -8.1607e-17, -5.2725e-16, 2.4121e-14,\n -9.1399e-16, 2.0141e-15, -2.8699e-15, -1.3234e-15, -1.4828e-14,\n -1.9200e-14, 1.6463e-15, -3.9797e-14, -3.7928e-14, -4.7651e-16,\n -6.7710e-16, -6.6742e-16, 1.8300e-15, -1.7948e-14, -3.2901e-15,\n 2.9889e-15, -1.0821e-14, 2.5688e-17, 1.7212e-15, -2.5576e-15,\n -7.9587e-17, 5.9388e-16, 2.8166e-16, -1.1576e-14, 2.6772e-15,\n 2.2510e-14, -1.4074e-15, 4.7396e-15, -2.9171e-15, -3.5655e-16,\n 6.6331e-15, 1.8961e-15, 9.0168e-15, -3.1618e-15, 5.3577e-15,\n -7.6131e-15, -3.7895e-14, 2.3690e-15, -1.7741e-14, 5.5736e-15,\n -4.5179e-15, -6.3226e-16, -4.0192e-16, -6.5112e-17, 1.8689e-14,\n -1.4779e-14, 2.9138e-15, -8.7825e-17, 1.7107e-15, -1.4360e-14,\n -1.4436e-15, -1.0306e-15, -7.7008e-17, -7.1480e-14, -6.1674e-16,\n -9.7126e-15, -2.4195e-14, 8.9959e-16, 1.5679e-15, 1.4983e-16,\n -4.0359e-15, 1.2838e-14, -7.1887e-16, -3.5018e-14, 3.3512e-15,\n -3.0740e-14, 4.3733e-16, -6.2813e-15, -4.9539e-15, 3.6042e-15,\n -1.4646e-15, -5.4301e-15, -3.5434e-14, -3.1241e-16, 6.3056e-17,\n -3.9013e-14, -1.6844e-14, -8.7338e-15, -4.9247e-16, -9.7400e-16,\n 1.1978e-14, -1.2021e-14, 2.3907e-15, 5.3697e-16, -8.2124e-15,\n -2.5649e-15, -8.2090e-15, -2.9693e-15, 2.1530e-15, -1.7004e-15,\n 1.0605e-15, 1.1035e-15, -8.4745e-15, -4.2487e-15, 1.0125e-16,\n 3.3466e-15, -5.2905e-14, -6.1237e-16, -1.3356e-14, 4.1851e-15,\n -1.7927e-15, -9.2173e-15, -3.1220e-16, -1.5244e-16, 8.8501e-16,\n 3.8821e-15, 1.0515e-15, 8.9413e-15, -8.7859e-17, -7.5068e-15,\n -1.7455e-15, 2.7873e-15, -1.1474e-15, -6.9582e-15, -1.0804e-15,\n -1.0003e-15, 1.4540e-15, -1.5551e-14, -3.3615e-15, 5.3384e-16,\n -2.6137e-15, -2.9438e-15, -1.8905e-14, 2.3748e-15, -5.5683e-14,\n -1.3836e-15, 3.5260e-15, -6.4911e-16, -1.8553e-15, -7.2729e-16,\n -1.7331e-15, -2.8626e-15, 2.3257e-15, -6.5508e-16, 5.5352e-16,\n -1.0618e-15, -1.2809e-14, 1.9728e-15, 1.5542e-15, 1.2074e-15,\n 2.5713e-15, 3.4909e-16, -5.4760e-15, 2.1688e-15, 5.4797e-16,\n 4.0501e-16, 9.9851e-15, 7.0349e-15, -2.5659e-15, -5.4146e-14,\n 4.1201e-15, -1.6326e-16, 1.3989e-15, 2.8856e-16, 3.2216e-15,\n -2.9873e-16, -1.1784e-15, -2.5326e-15, -2.1195e-15, -6.4439e-16,\n -3.5261e-14, 1.9780e-14, 9.8398e-15, 3.4255e-14, -1.9706e-14,\n 5.4743e-16, -2.6062e-14, 3.2042e-14, -5.7674e-15, -1.8833e-15,\n -1.1337e-15, -3.9686e-16, 9.1431e-16, 1.2206e-14, -1.1365e-15,\n -4.9311e-15, -8.3318e-15, -7.5176e-14, -1.1726e-14, -3.6820e-16,\n -2.9671e-14, -1.8843e-15, -1.6738e-14, 4.3937e-17, -1.0534e-14,\n 6.1691e-15, 1.1410e-14, -2.7148e-15, -9.3950e-16, -1.5394e-14,\n -3.8655e-14, -1.6503e-15, -3.9790e-14, 2.0376e-15, 6.8974e-15,\n -3.3640e-15, 2.3927e-16, 7.2298e-15, -3.0312e-15, -1.5567e-15,\n 6.9171e-16, 2.1599e-16, -1.5447e-14, -1.6736e-14, -2.4618e-16,\n 3.1502e-16, 2.3460e-15, 3.7081e-15, -2.4185e-15, -1.9838e-16,\n -1.8194e-15, -3.3265e-15, -1.2736e-15, -4.5316e-15, -6.5041e-16,\n -4.5234e-15, 1.7977e-15, -3.4604e-16, 5.7004e-15, 3.6862e-16,\n 6.8233e-17, -1.5201e-17, -2.1912e-15, -2.1876e-16, -2.0512e-16,\n 2.7102e-16, -2.3869e-14, -2.1136e-15, -1.3966e-15, 9.5040e-16,\n -4.9902e-14, -3.0473e-15, -6.0474e-15, -2.3218e-15, 7.0889e-14,\n 3.4285e-16, -2.2512e-15, -7.0433e-17, -1.1221e-15, -1.1104e-14,\n 2.4494e-16, 1.3835e-15, -4.1347e-16, -3.6590e-14, 3.6592e-15,\n 2.8312e-14, 8.3184e-16, -1.1362e-14, 9.0196e-15, 2.7798e-15,\n -6.4729e-16, -2.4846e-16, -5.7524e-16, 1.1023e-14, 7.6616e-15,\n -5.4300e-15, 1.0034e-15, -8.9689e-16, 4.7976e-16, 5.2591e-15,\n -5.1599e-15, -4.5760e-16, 2.6155e-14, -2.1690e-14, -1.9735e-15,\n -1.1404e-14, 2.5144e-15, -1.0436e-15, -1.7584e-15, -4.7946e-14,\n 4.9475e-15, -2.2120e-14, -7.5759e-16, -1.6413e-15, -1.7864e-14,\n 2.9142e-16, 3.3683e-15, -3.3311e-15, 7.6519e-16, 5.2669e-15,\n -1.5175e-16, -3.3193e-15, 2.8169e-15, 2.6978e-15, 2.6425e-15,\n 5.9853e-16, 2.7068e-15, 1.6113e-15, -1.0162e-16, -8.1253e-15,\n -1.9836e-14, 4.3907e-16, -1.8880e-17, 1.6603e-15, -1.5678e-16,\n -6.8239e-15, -2.0335e-17, -1.2560e-13, 3.5641e-15, -4.8549e-18,\n -1.0136e-14, 5.7629e-16, -7.8610e-17, 2.9201e-15, 1.6808e-15,\n -2.7377e-14, -3.6394e-15, 2.7855e-15, -1.0515e-14, -3.5883e-14,\n -2.1148e-14, 2.1186e-14, 5.9063e-15, 6.3712e-16, -1.1741e-14,\n -7.6516e-16, 3.9344e-16, 2.9780e-14, 1.8365e-16, -7.5430e-15,\n 1.7444e-15, 2.7030e-16, 1.6011e-14, 1.9241e-15, -6.5627e-14,\n -4.6932e-14, 1.5071e-14, 2.1808e-15, -9.2333e-14, -1.6549e-16,\n -7.6248e-14, 5.9733e-15, 1.0563e-14, -1.0643e-15, 1.2620e-14,\n 1.7438e-15, 4.0984e-17, -7.0883e-15, -5.3723e-16, -7.5789e-15,\n 1.1348e-14, -2.5836e-15, -5.3549e-15, 6.3785e-14, 2.5843e-15,\n 2.3076e-16, 8.0676e-15, -1.6643e-14, 5.5099e-14, -1.4064e-14,\n -4.3754e-16, 1.3709e-15, -2.9236e-15, -2.8645e-14, 1.5042e-15,\n -8.8812e-15, -1.2025e-14, 2.3325e-16, -2.8254e-16, -1.8920e-15,\n -2.5606e-14, 4.8590e-16, -2.3619e-14, 6.9202e-16, 3.1680e-15,\n -8.3288e-15, -1.2300e-16, -5.3962e-15, 3.3549e-17, -1.9282e-15,\n 2.4065e-14, -3.7809e-15, -2.8402e-15, 2.1581e-14, 4.3327e-15,\n -2.7327e-14, 3.0407e-15, -8.6181e-15, -1.2947e-14, 3.7557e-14,\n -1.7884e-14, -8.9462e-15, -1.1212e-14, 4.6103e-14, -3.8957e-15,\n 7.0303e-17, -1.6901e-14, 5.6704e-15, 1.4488e-15, -1.7844e-14,\n 1.5520e-14, -4.3274e-16, -5.6790e-15, -1.4273e-14, 1.1106e-15,\n -2.8336e-14, -7.4191e-15, -3.7565e-15, 8.5393e-16, 5.0931e-16,\n 2.2879e-15, -5.8339e-15, -4.8317e-14, -7.8304e-14, -8.9993e-16,\n -1.0770e-14, -1.0317e-15, 1.5956e-15, 1.8245e-15, -1.1122e-14,\n 3.2769e-14, -5.1679e-16, 2.9454e-15, -2.7031e-16, 1.5840e-15,\n -1.9518e-16, 1.2969e-16, -3.0405e-15, -1.2822e-14, -2.1070e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.2652e-12, 2.0007e-11, 6.8825e-14, 2.0355e-13, 8.0244e-13, 2.5908e-13,\n 1.0101e-10, 5.3827e-10, 4.4219e-13, 1.1368e-13, 8.7558e-12, 9.6850e-12,\n 7.6346e-10, 7.8014e-11, 7.2932e-11, 5.8426e-13, 3.4781e-10, 7.2495e-11,\n 1.3632e-11, 1.2068e-13, 6.8087e-12, 1.5231e-11, 2.2801e-11, 5.5814e-12,\n 9.5306e-13, 4.1213e-12, 6.2794e-12, 5.9299e-12, 5.6780e-13, 1.6134e-11,\n 1.4429e-10, 8.8697e-13, 2.7446e-10, 7.1343e-12, 3.7364e-12, 5.9261e-10,\n 2.0634e-10, 1.1354e-13, 5.1765e-13, 4.1313e-14, 1.0027e-13, 1.4640e-12,\n 3.4638e-10, 1.6762e-12, 3.1887e-11, 5.2261e-12, 1.7324e-11, 3.2017e-13,\n 1.4058e-13, 6.0804e-11, 9.1187e-13, 1.2231e-12, 2.2934e-14, 6.1819e-14,\n 3.2226e-13, 1.2763e-12, 5.9806e-13, 9.8480e-11, 5.7705e-10, 2.5045e-10,\n 7.4557e-13, 3.1920e-10, 3.6076e-10, 1.4929e-13, 3.2576e-12, 3.8768e-13,\n 7.9366e-10, 2.2048e-12, 2.0919e-12, 1.2406e-12, 1.0662e-11, 1.6861e-13,\n 5.6246e-12, 7.5070e-14, 4.5725e-14, 3.3591e-12, 7.1379e-12, 6.7185e-10,\n 5.4743e-13, 2.5668e-10, 2.9506e-11, 6.4082e-13, 5.5611e-14, 5.9110e-13,\n 1.3591e-13, 3.8790e-14, 5.1616e-13, 9.7439e-12, 7.0133e-10, 1.0478e-10,\n 4.1167e-13, 7.2887e-14, 2.1404e-10, 2.4516e-11, 1.7732e-13, 8.2880e-12,\n 3.5452e-11, 3.6406e-09, 6.4103e-13, 3.3985e-13, 1.5046e-11, 1.9749e-12,\n 6.3508e-11, 3.1243e-11, 1.4016e-10, 1.5378e-11, 1.2573e-12, 3.3455e-13,\n 4.3896e-10, 6.6804e-13, 1.0726e-11, 2.2511e-13, 3.7184e-10, 4.1798e-11,\n 1.9745e-11, 1.0264e-10, 1.7236e-12, 6.1973e-10, 2.7673e-12, 4.8801e-13,\n 3.5745e-10, 1.1549e-10, 6.5421e-13, 2.0284e-12, 1.2624e-10, 8.2353e-13,\n 3.5755e-12, 3.2673e-13, 1.0374e-11, 4.4586e-11, 2.6757e-10, 3.9791e-12,\n 1.6793e-10, 6.4538e-13, 1.7119e-10, 1.8275e-10, 2.8296e-13, 2.4308e-12,\n 3.9561e-11, 9.2410e-12, 7.2337e-12, 8.4177e-13, 7.6776e-11, 1.9768e-09,\n 2.5112e-13, 1.2054e-13, 5.2905e-12, 7.8513e-10, 1.0600e-12, 4.3672e-10,\n 4.9807e-14, 4.8295e-13, 3.6056e-11, 5.1616e-13, 7.7907e-13, 1.7446e-11,\n 7.2380e-13, 5.4164e-11, 1.5507e-13, 5.0436e-13, 2.9853e-10, 4.6679e-13,\n 8.6121e-10, 1.8193e-13, 3.6990e-11, 8.9155e-14, 2.6309e-12, 2.8115e-12,\n 6.4263e-13, 7.9508e-13, 2.6203e-11, 1.0180e-12, 2.9021e-12, 2.6744e-13,\n 2.5679e-12, 2.8918e-10, 2.9547e-13, 7.1607e-10, 1.0041e-13, 1.0145e-12,\n 6.9714e-12, 6.4783e-12, 2.5308e-11, 9.3792e-12, 4.0993e-12, 4.8372e-11,\n 1.8196e-09, 1.7888e-12, 5.8912e-10, 1.1284e-11, 2.3897e-12, 2.6721e-10,\n 4.6418e-13, 5.3919e-14, 1.7983e-12, 8.7818e-10, 3.4715e-13, 8.3538e-12,\n 2.5452e-12, 3.6567e-11, 5.9214e-13, 1.8911e-13, 7.1806e-10, 2.6528e-13,\n 3.5601e-12, 5.5339e-10, 1.2136e-12, 2.3378e-11, 5.7693e-13, 5.3460e-11,\n 2.1799e-13, 2.1464e-10, 5.0215e-12, 1.6659e-09, 4.3417e-11, 1.7096e-10,\n 6.1957e-11, 3.2263e-13, 2.2512e-13, 1.7865e-10, 2.0987e-12, 5.1134e-12,\n 4.5056e-10, 1.0654e-10, 1.6601e-10, 1.6292e-11, 2.0231e-10, 1.6622e-12,\n 5.1157e-10, 6.9815e-14, 1.1212e-13, 1.1476e-12, 3.8687e-11, 2.2326e-09,\n 3.7223e-11, 4.0140e-11, 1.1305e-10, 1.2032e-12, 5.9789e-12, 2.6949e-13,\n 8.4542e-13, 2.8830e-13, 3.1183e-11, 1.6358e-11, 2.3367e-12, 1.4667e-13,\n 5.9969e-12, 6.2581e-12, 3.2660e-13, 2.0506e-11, 3.4848e-10, 1.0170e-09,\n 1.1395e-12, 7.5913e-12, 3.0503e-12, 1.4343e-09, 3.9094e-10, 2.2979e-12,\n 5.1408e-10, 6.3856e-10, 1.3734e-13, 3.2416e-12, 8.5772e-11, 1.3029e-11,\n 4.3907e-10, 1.1386e-09, 1.4866e-12, 1.9110e-10, 2.4360e-11, 1.9984e-12,\n 2.6947e-12, 1.3628e-12, 9.8829e-11, 3.0546e-10, 1.8384e-11, 6.3220e-12,\n 1.8050e-11, 8.0943e-14, 1.2271e-09, 1.7873e-11, 2.0198e-11, 8.6735e-11,\n 4.3737e-14, 4.7685e-10, 1.5039e-11, 6.7242e-11, 2.4871e-13, 4.0479e-10,\n 1.1898e-11, 1.4151e-13, 1.5085e-13, 9.4293e-12, 2.6964e-10, 1.3866e-12,\n 1.3827e-12, 1.4554e-09, 3.2153e-10, 2.2182e-12, 2.3749e-13, 2.4170e-11,\n 5.0603e-11, 5.8027e-11, 1.5731e-09, 1.1660e-12, 6.1366e-10, 1.9245e-10,\n 1.0204e-12, 1.1370e-11, 5.2246e-14, 1.3687e-13, 8.2074e-14, 2.4673e-13,\n 2.2429e-11, 1.7478e-10, 1.8628e-11, 3.7557e-14, 1.3143e-13, 5.3123e-13,\n 2.5988e-11, 1.7997e-11, 1.2800e-12, 5.7657e-11, 4.1578e-12, 2.4215e-13,\n 2.5125e-13, 8.6645e-13, 7.2737e-12, 3.6179e-12, 1.5822e-12, 1.5857e-11,\n 7.7716e-12, 2.1276e-11, 3.5312e-11, 5.0744e-13, 7.6487e-12, 1.9046e-12,\n 2.6217e-12, 1.2908e-11, 5.1113e-13, 1.1639e-09, 9.5471e-12, 2.2771e-13,\n 2.1973e-11, 2.9517e-12, 8.5773e-13, 2.8202e-13, 6.6164e-13, 2.5216e-12,\n 1.1885e-11, 4.3857e-11, 2.1390e-14, 3.6107e-12, 1.3433e-12, 4.2349e-11,\n 3.6086e-13, 2.0875e-12, 4.5528e-12, 2.3782e-12, 3.5403e-12, 3.3668e-13,\n 4.7392e-10, 4.5399e-11, 1.9802e-12, 2.6189e-13, 2.2521e-12, 2.9565e-13,\n 1.4675e-11, 3.2992e-10, 6.8391e-13, 7.1151e-10, 1.3032e-11, 2.0776e-14,\n 2.2384e-11, 3.4414e-09, 4.2229e-14, 1.3013e-13, 5.2806e-13, 1.7013e-12,\n 1.3275e-09, 4.0777e-12, 4.1247e-12, 2.3327e-12, 3.8332e-13, 3.2159e-11,\n 3.7057e-11, 6.9059e-13, 2.8904e-10, 1.5327e-10, 1.7421e-11, 8.7104e-14,\n 1.8828e-12, 8.5679e-13, 7.7096e-10, 2.0168e-11, 4.7643e-10, 1.0051e-13,\n 1.8893e-12, 4.5574e-10, 7.8198e-10, 1.3042e-11, 5.7452e-11, 4.9988e-10,\n 1.0509e-10, 2.3099e-12, 4.0588e-13, 9.0482e-12, 1.6689e-13, 7.6073e-13,\n 4.3394e-11, 1.8620e-10, 5.8231e-13, 1.1440e-10, 3.4820e-11, 9.2117e-13,\n 5.4542e-14, 1.1734e-12, 2.1964e-13, 2.6397e-10, 2.0931e-12, 3.1579e-11,\n 6.5569e-13, 4.4200e-10, 1.1949e-09, 6.0783e-11, 1.5484e-10, 9.4966e-12,\n 3.0599e-13, 5.7320e-11, 2.0324e-12, 1.6903e-11, 2.0242e-11, 7.6103e-11,\n 3.9864e-12, 3.7029e-10, 1.3752e-13, 1.5231e-12, 1.7846e-11, 1.4421e-12,\n 4.6751e-11, 1.9329e-11, 8.6690e-12, 2.0154e-12, 3.6585e-13, 3.1054e-10,\n 1.3652e-12, 4.8979e-10, 4.9261e-10, 6.3129e-12, 2.1967e-13, 3.1710e-13,\n 4.9834e-12, 4.6873e-10, 7.3947e-12, 1.7739e-12, 1.2127e-11, 2.0518e-11,\n 9.9834e-14, 1.5566e-13, 3.4881e-10, 4.0332e-11, 5.7216e-13, 2.4441e-13,\n 2.0857e-12, 1.0625e-12, 2.9399e-13, 5.9801e-11, 1.2587e-10, 1.5982e-13,\n 1.1160e-10, 2.2180e-11, 1.1395e-11, 1.2680e-11, 8.4538e-12, 1.1716e-10,\n 7.4459e-12, 1.1882e-11, 3.5129e-13, 7.0363e-12, 9.5864e-12, 2.1409e-12,\n 6.8293e-11, 8.3892e-13, 5.1994e-10, 1.1729e-11, 4.4707e-11, 1.0246e-10,\n 4.0322e-11, 1.1908e-12, 1.9950e-13, 2.3838e-10, 1.3061e-11, 8.3174e-10,\n 2.2000e-12, 8.8397e-13, 5.7991e-11, 2.2756e-13, 3.9637e-13, 1.9160e-11,\n 5.2810e-11, 1.7617e-10, 3.0254e-12, 1.4763e-12, 6.1826e-13, 1.0300e-11,\n 7.9873e-13, 8.3380e-13, 1.1607e-13, 5.1959e-13, 9.3645e-13, 1.0384e-10,\n 3.4069e-10, 4.8628e-11, 5.4513e-12, 9.4421e-13, 8.0049e-10, 5.8705e-10,\n 5.2987e-13, 6.9799e-12, 6.2985e-10, 7.1297e-14, 2.1385e-12, 1.0857e-13,\n 8.4561e-11, 2.8674e-11, 3.1107e-12, 3.3585e-10, 2.4901e-11, 3.9602e-13,\n 2.6209e-11, 4.2069e-13, 2.7811e-12, 9.5318e-12, 3.0336e-10, 9.2721e-13,\n 2.3736e-10, 1.7958e-11, 5.9821e-12, 7.4893e-12, 7.9566e-14, 1.2775e-10,\n 2.9575e-12, 3.2543e-12, 6.5216e-10, 4.3490e-10, 1.4071e-11, 1.3579e-11,\n 7.9801e-12, 2.2235e-12, 5.9390e-10, 1.4788e-13, 9.7725e-12, 8.1125e-12,\n 1.7028e-09, 8.9260e-12, 1.3152e-09, 2.2508e-11, 2.1845e-13, 1.5303e-12,\n 5.0885e-13, 1.1854e-11, 2.6427e-12, 7.1817e-10, 6.1177e-11, 4.4817e-12,\n 1.8698e-10, 1.6576e-13, 1.9962e-09, 1.4604e-11, 7.3679e-11, 1.8611e-12,\n 5.9478e-11, 3.8688e-12, 3.6976e-13, 3.7167e-10, 1.2998e-12, 5.3882e-12,\n 7.2334e-11, 2.6801e-14, 2.2665e-13, 1.4955e-10, 4.0618e-12, 7.0825e-14,\n 3.6068e-10, 2.7379e-13, 4.7435e-13, 3.7601e-13, 7.4623e-15, 4.8736e-13,\n 7.4830e-13, 1.1919e-11, 9.2876e-10, 5.7300e-11, 1.5702e-12, 4.3654e-11,\n 1.1520e-11, 3.8160e-10, 7.8756e-14, 4.9047e-12, 1.7677e-12, 7.7931e-11,\n 5.0816e-11, 1.1097e-10, 6.0224e-12, 2.6307e-14, 1.3345e-12, 3.8956e-13,\n 1.3163e-12, 9.2904e-10, 2.1201e-10, 9.5471e-10, 1.3694e-13, 1.7894e-13,\n 9.2423e-13, 3.6896e-12, 5.6981e-12, 1.2359e-10, 2.6345e-12, 4.4581e-13,\n 3.9559e-14, 3.4174e-12, 1.3848e-12, 7.2501e-12, 3.4038e-11, 1.8510e-12,\n 2.3369e-11, 9.1032e-11, 1.7482e-11, 2.3019e-12, 5.8112e-12, 4.2468e-12,\n 1.3060e-10, 3.0554e-13, 4.7437e-13, 3.1941e-13, 9.4815e-12, 7.6255e-11,\n 1.5885e-13, 3.7940e-12, 3.2586e-12, 3.4775e-10], device='cuda:0')" + }, + "49": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 5.8338e-15, -9.8782e-14, 2.5961e-14, 8.8791e-16, 2.6462e-15,\n -2.3703e-14, 4.5213e-16, -1.6104e-15, 1.1044e-14, -2.8474e-15,\n -1.4042e-15, 1.7627e-15, -3.7673e-14, -6.7023e-15, -1.4311e-14,\n 3.4539e-15, -2.7643e-14, 1.6122e-14, 1.5500e-14, -2.9358e-15,\n 1.1603e-15, -9.6702e-15, 2.4702e-15, 1.6176e-14, -6.9367e-15,\n -1.6998e-14, 3.7545e-15, 2.1500e-14, -4.2422e-15, -2.3243e-14,\n 3.0547e-15, 2.4950e-15, 1.2898e-14, -2.5394e-14, -2.9632e-15,\n -1.0056e-14, -1.3688e-14, -2.5170e-15, -4.6898e-14, 2.6253e-16,\n 3.2062e-15, 5.4323e-16, -7.5452e-15, -2.6390e-14, -3.5694e-14,\n -1.5105e-14, 8.4618e-17, -3.8509e-15, -1.5825e-15, -2.1648e-14,\n -9.0076e-16, 4.0413e-16, 1.8435e-16, 1.8953e-15, 6.8332e-16,\n -1.3712e-15, -1.8313e-14, 1.3842e-14, -8.6771e-15, -3.2599e-15,\n 4.0624e-15, -6.7247e-15, 5.0639e-15, -1.2652e-14, 1.3140e-15,\n -1.6934e-15, -2.5807e-14, 2.1806e-14, -6.7372e-15, -4.5820e-15,\n -1.6095e-14, -4.5635e-15, 1.0428e-15, -1.3590e-15, -1.1619e-15,\n 1.5287e-15, -1.6218e-14, 8.7000e-15, -2.4924e-14, -1.2273e-14,\n 2.2154e-14, 1.0883e-15, 2.2164e-14, 6.8140e-17, -3.7212e-15,\n -3.0793e-14, -1.2380e-14, -2.2092e-14, -1.6681e-14, 1.7477e-15,\n 2.6201e-15, 6.2747e-15, -2.1623e-14, 3.1994e-16, 2.4114e-15,\n 4.1660e-15, -2.3558e-14, -1.3577e-14, 1.5093e-15, 9.7485e-15,\n -2.6767e-14, -3.3062e-15, -1.8547e-14, -1.0530e-14, 4.4461e-15,\n -5.7353e-15, -9.9121e-16, 1.0334e-14, -9.0705e-15, -7.7653e-15,\n 4.8573e-15, -6.8370e-15, -6.9113e-15, -3.6551e-14, 2.8318e-14,\n 4.9058e-15, 4.3615e-15, 2.9504e-14, -2.8321e-14, 6.2598e-15,\n -1.2905e-14, 9.0266e-15, 6.8740e-15, 8.6091e-17, 7.6266e-18,\n -7.8046e-16, -6.9561e-14, -2.9490e-15, 6.1705e-15, 1.2503e-14,\n 3.0921e-15, 1.8944e-15, 6.4313e-15, 4.3878e-16, -3.2951e-15,\n -2.8913e-14, -9.3872e-17, -6.2954e-16, 2.2997e-16, 7.0822e-17,\n 2.7118e-16, -1.6096e-14, 9.7956e-15, -3.7197e-15, 7.1967e-15,\n -3.8560e-16, 5.6389e-15, -6.8218e-15, -3.2298e-15, -1.0508e-14,\n 1.3459e-15, -1.2284e-14, 8.4265e-16, -1.2427e-14, 2.5319e-15,\n 1.6105e-15, 8.6413e-15, -2.6723e-14, -1.7614e-15, 3.1207e-16,\n 1.0962e-14, -1.1887e-14, -2.4110e-14, 6.1221e-17, -1.2634e-14,\n 9.8324e-15, -8.7747e-16, -2.1682e-15, -1.3395e-16, 1.9289e-14,\n 1.3663e-14, -1.9673e-15, -3.1083e-15, -2.6861e-14, -2.8462e-14,\n 1.3001e-14, -3.4827e-15, -1.2215e-15, 9.1404e-17, -7.3210e-15,\n 2.4935e-15, 1.1540e-14, -4.4690e-14, 1.5302e-15, -3.3968e-14,\n 2.7614e-14, -2.8813e-14, -1.4266e-15, -2.4929e-14, 1.4404e-14,\n 1.4992e-14, -1.3120e-14, 1.5740e-15, -7.9837e-16, -2.5051e-15,\n -1.6440e-14, 7.8593e-15, -3.1266e-15, 1.6746e-16, -2.8341e-14,\n -1.8375e-15, 7.0492e-16, -8.0029e-15, -1.1396e-14, 5.9797e-15,\n 1.3576e-14, -1.1634e-14, -4.7747e-14, 1.8230e-16, -5.5563e-16,\n -1.9913e-16, -2.6327e-15, -1.0160e-15, -4.4510e-15, 4.8145e-15,\n 4.0887e-14, -3.5012e-15, 3.7204e-15, 1.7535e-15, 3.6055e-14,\n 7.6677e-16, -1.5698e-15, -5.6393e-15, -8.8839e-15, -1.1737e-14,\n -1.8738e-14, 3.1040e-15, -2.6789e-14, -4.1478e-14, 6.5374e-16,\n -3.5961e-15, 1.2305e-15, 1.3982e-15, -2.4979e-14, 2.7286e-15,\n -2.8933e-15, -1.2874e-14, -9.8592e-16, -5.1842e-16, 8.9287e-15,\n -1.6053e-15, -5.9429e-16, 6.7384e-17, -1.2864e-14, 3.4341e-15,\n 4.2544e-14, -3.5981e-15, -3.5077e-15, 2.0762e-15, -1.1838e-14,\n 7.2502e-15, 7.5521e-15, 3.1640e-14, 2.6315e-15, -1.8010e-16,\n -9.4476e-15, -2.7077e-14, -1.3745e-15, -2.8499e-14, 3.6745e-15,\n -1.5720e-14, -4.5525e-15, -6.5084e-15, -3.1571e-15, 2.2274e-14,\n -1.1762e-14, -2.6662e-15, -1.6505e-15, 1.3684e-14, -4.4717e-14,\n 1.2503e-15, 1.2354e-15, -1.7254e-15, -4.3068e-14, 6.8370e-15,\n -2.0714e-14, -3.5169e-14, 2.9357e-15, 1.0778e-14, -1.2916e-14,\n -9.4124e-16, 2.7903e-14, -9.7687e-15, -5.6504e-14, 2.5882e-15,\n -4.4011e-14, -1.3298e-15, -1.4718e-14, 2.3744e-15, 9.4011e-15,\n -5.1953e-15, 2.6513e-15, -3.2836e-14, 3.8570e-16, 3.2730e-15,\n -4.2967e-14, -1.8690e-14, 5.7169e-15, 3.1985e-16, -8.8019e-15,\n 1.0077e-14, -1.9973e-14, 2.0506e-15, -1.9473e-15, -2.9773e-14,\n -3.2416e-15, -8.8862e-15, -1.0125e-14, -9.5117e-15, 1.5309e-15,\n -2.2142e-15, 1.6147e-15, 5.8366e-15, 1.7085e-15, -1.0436e-16,\n -9.9704e-15, -3.8695e-14, -3.8466e-16, -2.8213e-14, -2.7552e-15,\n 8.8053e-16, -7.8796e-15, 2.9391e-16, 8.7687e-17, 1.2214e-15,\n -4.8471e-15, -9.3813e-16, 8.6772e-15, -1.5481e-15, -1.8216e-14,\n -9.4103e-15, -5.7352e-16, -6.0630e-15, -1.4067e-14, 8.7563e-16,\n 7.5703e-15, -1.1471e-15, -6.0997e-15, -3.7253e-15, 3.6179e-15,\n -8.5439e-15, 1.3131e-14, -2.9133e-14, -7.7256e-17, -5.7893e-14,\n 8.3327e-15, 1.3478e-14, -3.7970e-17, 5.7663e-16, -7.2143e-15,\n 4.8650e-15, 1.6983e-15, -2.3450e-15, -3.4480e-16, 6.0047e-15,\n 6.5131e-16, -1.9554e-14, 8.7388e-15, -5.9099e-15, -5.5710e-15,\n -2.4366e-15, -1.6096e-14, -2.2803e-15, -1.1769e-14, -1.3113e-15,\n -3.8805e-16, 1.0141e-14, 5.5540e-15, 2.0340e-15, -3.6799e-14,\n 1.3526e-14, 1.3794e-16, -4.8552e-16, 1.7929e-15, -3.4267e-15,\n 3.8836e-16, 1.2483e-15, -1.2795e-14, -2.9373e-15, 2.2454e-15,\n -3.4537e-14, 3.6386e-14, 3.3796e-14, 1.6220e-14, -1.4699e-14,\n -6.1523e-16, -3.0609e-14, 2.4842e-14, 4.0418e-15, 1.5930e-15,\n -1.2631e-15, 1.0741e-15, 1.8780e-15, 3.1141e-14, -2.2318e-14,\n -7.0118e-15, 5.1200e-15, -5.9277e-14, -3.6455e-14, -7.3238e-15,\n -3.2974e-14, -3.4493e-15, -2.1989e-14, -7.7549e-17, -1.4912e-14,\n -3.5955e-15, 1.0071e-14, 1.3582e-15, 2.9487e-16, -1.6827e-14,\n -2.9891e-14, -7.5940e-15, -3.9486e-14, -6.2962e-16, 4.3617e-15,\n -2.3170e-14, 1.0432e-16, 2.3934e-14, -6.9576e-15, -1.0289e-14,\n -1.8448e-16, -7.1916e-15, -2.3159e-14, -2.5787e-14, -1.4257e-15,\n -5.7491e-17, -9.3362e-15, 8.2818e-15, 1.0267e-15, 1.8310e-16,\n -1.8730e-15, -3.3366e-15, 6.0374e-15, -3.4051e-15, 5.7342e-15,\n -2.5815e-14, 3.7788e-15, -1.7927e-14, 8.7796e-15, -4.0086e-15,\n -6.0589e-15, 4.9376e-17, 2.8641e-15, -5.2816e-15, 1.1327e-16,\n -1.2327e-15, -2.8576e-14, -4.6249e-15, -3.9819e-15, -1.1392e-15,\n -2.7675e-14, -1.0042e-14, -9.1696e-15, 8.9584e-16, 6.2895e-14,\n 1.6436e-15, 2.2016e-15, -1.6198e-15, -2.1132e-15, -1.3089e-14,\n -1.7705e-15, -1.0874e-15, 4.2099e-16, -3.5778e-14, -3.5819e-15,\n 2.7357e-14, 1.3747e-15, 5.9487e-15, 5.9565e-15, 1.6039e-14,\n 4.3509e-15, 1.6360e-15, 2.1455e-16, 1.4083e-14, 2.1513e-14,\n -2.0040e-14, 3.9803e-15, 1.2564e-15, -1.4161e-15, -1.4008e-15,\n -5.9618e-15, 4.4723e-15, 1.9363e-14, -1.5658e-14, -3.1000e-15,\n -1.9959e-14, 4.7149e-15, 6.9668e-16, -6.9990e-15, -6.0798e-14,\n -3.1010e-15, -1.9558e-14, 1.2877e-16, 9.7700e-16, -1.2681e-14,\n -2.5876e-16, -2.3625e-15, 2.3032e-15, -1.0278e-14, 5.8830e-15,\n 6.5437e-16, -5.1117e-15, -1.7030e-15, -3.2015e-15, 6.2433e-15,\n -5.5168e-16, -7.1788e-15, -5.1867e-15, -6.1632e-16, -1.4772e-14,\n -2.0780e-14, 1.8613e-15, 6.9384e-16, -5.4485e-16, 4.0961e-16,\n -1.0565e-14, -1.5465e-16, -5.0961e-14, 5.4014e-15, -2.4062e-18,\n -1.9246e-14, -4.4645e-16, 2.1094e-16, 1.2460e-14, -5.4825e-15,\n -1.6361e-14, -1.1159e-14, 9.4367e-15, -2.1030e-14, -2.5815e-14,\n -2.3125e-14, 3.2424e-14, -5.0097e-17, -7.3541e-16, -1.0811e-14,\n 3.3441e-17, 7.1293e-15, 4.6703e-14, -3.1745e-17, -1.2622e-14,\n -1.4045e-15, -8.2455e-15, 2.0479e-14, -4.2555e-15, -2.3068e-14,\n -4.6371e-14, 2.2599e-14, 3.2111e-15, -5.8719e-14, 6.0769e-15,\n -3.7385e-14, -5.0296e-15, 2.0079e-14, 5.1200e-16, 9.1358e-15,\n -1.2208e-15, -4.1130e-17, 4.0630e-15, -1.5632e-16, -5.7449e-15,\n 1.3067e-14, -2.4579e-15, -1.0993e-14, 3.8758e-14, 6.2465e-15,\n -2.6080e-15, 6.5660e-15, -3.5306e-14, 3.8960e-14, -2.1632e-14,\n 1.8585e-15, -1.1085e-15, -4.8521e-15, -3.7066e-14, -1.9766e-14,\n -1.5673e-14, -1.2608e-14, -3.8184e-15, 3.2437e-16, -5.0406e-15,\n -5.1919e-14, -1.3250e-15, -2.7097e-14, -7.8489e-15, -4.1735e-15,\n -2.5394e-14, -1.5130e-14, 2.7613e-15, -3.3583e-16, 1.2886e-15,\n 1.7801e-14, -4.5717e-15, 2.1127e-15, 2.3743e-14, 5.4639e-15,\n -2.6937e-14, 5.6971e-15, -2.3461e-14, -2.3150e-14, 3.1376e-14,\n -1.9100e-14, -7.6396e-15, -9.8315e-15, 3.3149e-14, -5.2526e-16,\n -2.1301e-17, -3.6493e-14, 1.8945e-14, 3.6614e-15, -2.8559e-14,\n 2.8361e-14, 2.3610e-16, -5.8838e-15, -1.0200e-14, -1.5134e-15,\n -1.8976e-14, -2.9046e-14, -4.4408e-15, 2.3383e-15, 7.5181e-16,\n -2.1037e-15, -2.9855e-14, -2.6215e-14, -2.5711e-14, 9.0064e-15,\n -1.3180e-14, 1.1209e-15, -1.3423e-14, -2.3608e-15, -2.6440e-14,\n 2.9996e-14, 8.2321e-15, -2.2061e-15, -1.5543e-15, -1.8953e-15,\n -2.8413e-15, -8.7237e-15, -5.4020e-15, -1.5413e-14, -2.8796e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.7484e-11, 1.6832e-11, 9.7160e-13, 2.1947e-12, 2.8701e-13, 3.2289e-12,\n 2.6222e-10, 5.3165e-10, 4.8182e-12, 1.6995e-12, 1.1562e-10, 1.5400e-10,\n 4.3880e-10, 5.3623e-11, 3.1458e-11, 1.3291e-11, 2.8080e-10, 3.9420e-10,\n 1.1374e-10, 1.3514e-12, 3.8773e-11, 1.7494e-11, 2.4724e-11, 2.5146e-11,\n 1.9719e-11, 5.1648e-12, 3.6615e-12, 1.0746e-10, 1.2536e-11, 1.0063e-10,\n 5.7273e-10, 1.4594e-11, 3.1319e-10, 4.9860e-12, 1.3743e-10, 1.7265e-10,\n 1.8242e-10, 3.7784e-13, 1.9561e-11, 5.0564e-13, 3.6252e-12, 1.4158e-12,\n 2.9263e-10, 1.4156e-11, 5.4292e-11, 8.6327e-11, 6.9024e-12, 1.9707e-13,\n 2.9033e-12, 1.8288e-10, 6.9931e-13, 8.9930e-13, 2.8758e-13, 5.3828e-13,\n 6.4080e-12, 2.6984e-11, 9.7345e-12, 2.5325e-10, 5.2644e-10, 1.3855e-10,\n 3.9172e-13, 7.3141e-11, 4.7245e-10, 3.9026e-12, 1.0151e-12, 1.2865e-11,\n 6.7265e-10, 6.7880e-11, 1.0692e-12, 3.6885e-11, 1.9604e-10, 9.1554e-13,\n 2.8310e-12, 1.8978e-13, 1.5407e-12, 1.3493e-12, 1.8995e-11, 6.4712e-10,\n 1.9374e-12, 7.2863e-10, 6.6060e-11, 2.9582e-13, 2.0437e-12, 2.4086e-12,\n 4.6391e-12, 1.7321e-13, 1.1674e-11, 1.1837e-11, 2.5214e-10, 2.5016e-10,\n 1.3761e-11, 6.1857e-13, 2.9451e-10, 2.9626e-10, 1.4880e-13, 3.2786e-12,\n 3.6099e-11, 1.0310e-09, 6.7516e-12, 2.2584e-11, 2.0533e-11, 3.1893e-11,\n 1.0503e-10, 2.6419e-11, 1.7340e-10, 1.2155e-10, 7.4820e-13, 7.2400e-12,\n 1.8759e-10, 7.3144e-12, 4.5946e-12, 9.0634e-12, 7.0215e-10, 5.6776e-11,\n 3.3719e-11, 7.1067e-11, 2.7665e-11, 1.5646e-09, 8.6195e-12, 2.8419e-13,\n 3.2960e-10, 2.5682e-10, 2.3938e-11, 3.9168e-11, 2.5192e-10, 5.3152e-13,\n 1.6437e-11, 2.2462e-13, 8.4591e-11, 1.2209e-10, 7.4646e-11, 6.3459e-12,\n 4.5766e-10, 9.3070e-12, 7.1363e-10, 4.8161e-10, 1.8148e-11, 1.1871e-12,\n 2.3495e-10, 4.8178e-12, 2.7726e-12, 1.7418e-12, 1.8613e-10, 3.9844e-10,\n 3.3403e-12, 4.0930e-12, 1.0812e-11, 1.2714e-09, 1.2530e-11, 2.6178e-10,\n 4.1046e-14, 2.8246e-12, 2.4700e-10, 1.2350e-11, 3.7436e-13, 2.0241e-10,\n 2.1369e-12, 1.9032e-10, 1.0528e-13, 1.9388e-13, 6.0505e-10, 2.8700e-13,\n 2.8973e-10, 5.3478e-12, 7.3936e-11, 2.5612e-12, 2.0525e-12, 1.0086e-10,\n 1.9076e-13, 2.5992e-11, 3.7353e-11, 7.7228e-11, 3.2125e-11, 3.9223e-12,\n 3.7645e-11, 2.5055e-10, 3.6640e-12, 4.6861e-10, 2.8778e-14, 3.1388e-11,\n 3.4405e-11, 1.4671e-11, 1.3245e-10, 1.0130e-10, 1.7511e-11, 5.4215e-11,\n 5.7944e-10, 4.1607e-11, 3.8423e-10, 3.3491e-11, 1.0789e-10, 8.5051e-11,\n 1.1185e-11, 5.3122e-13, 2.4434e-11, 3.3796e-10, 3.6139e-12, 1.3506e-11,\n 3.5767e-11, 2.2822e-10, 2.2566e-12, 1.5162e-12, 8.8947e-10, 8.5258e-12,\n 1.5506e-12, 4.0234e-10, 1.5531e-11, 1.6847e-11, 2.0182e-11, 2.0648e-11,\n 3.5382e-13, 2.4482e-10, 1.4057e-11, 2.6993e-10, 3.9405e-10, 5.1696e-11,\n 1.8081e-10, 6.1739e-12, 1.6132e-11, 9.0061e-10, 2.7106e-12, 1.9186e-12,\n 4.5017e-10, 1.2639e-10, 5.8245e-11, 1.6583e-11, 4.0403e-10, 2.8246e-12,\n 1.4368e-10, 8.1996e-13, 2.0170e-12, 5.7627e-12, 2.6945e-10, 6.0171e-10,\n 1.8233e-11, 1.4358e-11, 1.5037e-10, 2.8054e-11, 3.5182e-11, 1.9931e-12,\n 5.0340e-13, 4.7909e-13, 1.5167e-11, 1.8741e-11, 2.0162e-11, 1.5831e-12,\n 6.6679e-11, 3.3933e-12, 6.2089e-12, 1.4341e-10, 3.2846e-10, 4.1587e-10,\n 7.6355e-12, 3.8542e-12, 3.7485e-11, 9.2655e-10, 9.8199e-11, 7.3197e-13,\n 1.5806e-10, 3.8136e-10, 4.9574e-13, 1.0744e-11, 5.3132e-11, 7.1229e-11,\n 5.5561e-10, 7.4524e-10, 8.0171e-13, 1.6125e-10, 3.8371e-10, 1.5679e-11,\n 1.1295e-12, 6.8593e-12, 2.1939e-10, 8.3495e-11, 2.2748e-10, 8.9096e-11,\n 3.1573e-11, 1.4955e-12, 6.5717e-10, 1.5509e-10, 2.1211e-11, 3.2423e-10,\n 9.1349e-13, 5.1787e-10, 2.1170e-11, 3.2994e-11, 4.6475e-13, 2.4200e-10,\n 4.0628e-12, 3.9358e-13, 6.7031e-12, 5.0182e-12, 3.4415e-10, 7.3764e-12,\n 1.2114e-11, 5.9549e-10, 8.0395e-10, 1.3765e-12, 9.3113e-14, 1.2104e-11,\n 5.5448e-11, 7.2368e-11, 7.2716e-10, 4.2510e-11, 2.8900e-10, 7.5740e-11,\n 2.1055e-11, 2.7513e-10, 7.3385e-13, 6.4624e-14, 1.3017e-12, 7.3873e-12,\n 8.7831e-12, 4.8563e-10, 7.9820e-12, 6.2259e-13, 1.4114e-12, 2.0795e-11,\n 2.9874e-10, 5.2064e-12, 7.1701e-12, 1.0009e-10, 5.8457e-12, 2.1192e-12,\n 6.9556e-12, 7.8169e-13, 2.7788e-12, 6.2274e-12, 7.8567e-12, 2.1373e-10,\n 2.5881e-11, 5.8491e-11, 1.1808e-10, 8.0856e-12, 3.7522e-12, 6.9396e-12,\n 2.2048e-12, 9.8787e-12, 1.0708e-11, 7.4733e-10, 1.0536e-10, 1.7716e-11,\n 9.7343e-11, 1.2729e-12, 5.6620e-11, 1.2910e-11, 3.6653e-11, 1.1446e-10,\n 3.0218e-12, 2.0787e-10, 9.6816e-13, 1.2046e-12, 6.9103e-13, 8.7298e-11,\n 3.9442e-13, 1.2805e-12, 8.5133e-11, 5.1537e-11, 2.3118e-11, 5.3962e-12,\n 5.9674e-10, 2.2279e-10, 1.1156e-11, 1.2614e-11, 1.3661e-10, 9.7041e-12,\n 1.6994e-11, 1.9596e-10, 1.3917e-11, 4.0612e-10, 2.2839e-10, 1.7635e-13,\n 5.3678e-11, 6.0284e-10, 7.3997e-13, 8.4547e-13, 1.9154e-11, 4.9307e-11,\n 6.2834e-10, 5.6492e-11, 4.1783e-11, 2.6363e-11, 7.0729e-12, 2.6677e-11,\n 1.6686e-11, 2.0428e-11, 2.4869e-10, 7.0433e-11, 6.2329e-12, 5.1240e-13,\n 4.1309e-11, 5.0809e-12, 8.8358e-10, 1.7434e-10, 1.9922e-10, 4.2506e-13,\n 1.0352e-12, 2.2739e-10, 1.4988e-09, 2.2042e-10, 1.3965e-10, 3.4821e-10,\n 6.0089e-10, 4.5487e-11, 2.4386e-12, 8.4713e-11, 1.3572e-12, 4.3487e-13,\n 1.2984e-11, 6.8255e-11, 2.6336e-12, 5.7733e-11, 4.7829e-11, 3.4698e-13,\n 9.7984e-13, 2.0066e-11, 2.7729e-12, 2.2399e-10, 8.8861e-11, 1.6084e-11,\n 5.1460e-12, 2.5270e-10, 3.8739e-10, 3.3610e-11, 5.2821e-10, 4.1077e-12,\n 6.3195e-12, 3.1940e-10, 1.6748e-12, 1.3036e-10, 7.7619e-11, 3.3776e-10,\n 1.5287e-12, 2.5633e-10, 1.9407e-12, 2.6722e-11, 1.4001e-10, 6.4174e-12,\n 4.8881e-11, 1.4918e-11, 2.6781e-11, 2.0668e-12, 3.4352e-13, 6.6812e-10,\n 1.3117e-12, 6.2087e-10, 8.4954e-10, 7.0188e-11, 4.3372e-13, 5.4532e-13,\n 5.5858e-12, 2.3879e-10, 2.2012e-11, 5.9875e-13, 2.6254e-11, 2.9436e-11,\n 1.7979e-12, 4.0810e-13, 7.7825e-11, 3.1738e-11, 7.7838e-12, 1.2119e-12,\n 1.3018e-12, 1.1489e-11, 4.2755e-13, 2.7064e-11, 1.3779e-10, 8.0292e-14,\n 6.2832e-11, 1.0517e-10, 1.9326e-11, 5.2578e-12, 3.1698e-12, 1.0180e-10,\n 1.1340e-10, 1.4872e-10, 5.2746e-12, 1.4589e-11, 1.1494e-11, 6.0245e-11,\n 6.7635e-11, 3.0805e-12, 2.1134e-10, 1.9055e-11, 5.8586e-11, 2.3496e-10,\n 2.3946e-10, 6.9368e-13, 2.6195e-12, 5.9774e-10, 4.9213e-12, 7.4266e-10,\n 1.0391e-12, 3.6866e-13, 1.7012e-10, 6.8467e-13, 1.1728e-11, 8.9984e-12,\n 1.8926e-10, 5.3341e-11, 2.5771e-12, 3.0269e-11, 2.4395e-12, 9.1799e-11,\n 9.3560e-12, 1.2978e-11, 1.0185e-12, 1.4665e-11, 2.5069e-12, 1.0656e-10,\n 1.4032e-10, 2.6322e-11, 1.2740e-11, 5.0264e-13, 1.7011e-09, 5.0127e-10,\n 1.0703e-11, 6.1998e-12, 9.9001e-10, 1.6799e-13, 4.5798e-11, 1.4058e-12,\n 8.2986e-11, 1.0323e-10, 1.7269e-11, 9.2605e-11, 1.9207e-11, 2.2840e-11,\n 1.2934e-10, 8.2000e-12, 3.8979e-11, 1.8775e-10, 3.5117e-10, 2.8135e-13,\n 1.7925e-10, 8.7047e-12, 1.3770e-10, 2.1156e-10, 6.9210e-13, 1.8209e-10,\n 3.7326e-12, 4.9752e-11, 4.5428e-10, 1.4610e-10, 8.5012e-12, 2.3285e-11,\n 5.1776e-11, 2.4058e-12, 4.6441e-10, 5.4444e-12, 1.2711e-11, 1.3250e-11,\n 6.2380e-10, 2.5412e-12, 1.4364e-09, 1.0366e-11, 8.6483e-12, 1.5955e-12,\n 2.8413e-13, 8.3895e-11, 7.4279e-12, 4.9901e-10, 1.2581e-10, 3.1828e-12,\n 8.7328e-10, 3.1448e-12, 6.5922e-10, 2.0692e-10, 1.2292e-10, 6.3979e-11,\n 4.3005e-11, 5.3864e-11, 1.7842e-11, 4.9027e-10, 1.2442e-11, 8.9353e-12,\n 2.1700e-10, 4.9596e-13, 1.0587e-13, 7.9846e-10, 3.3491e-11, 7.9505e-13,\n 6.7019e-10, 4.9748e-12, 9.4468e-13, 1.2707e-11, 5.7515e-14, 2.1614e-13,\n 1.2418e-12, 4.1710e-12, 5.2811e-10, 4.0293e-11, 6.8838e-13, 8.8266e-11,\n 1.3759e-11, 3.1818e-10, 3.6847e-12, 9.4049e-12, 7.8170e-12, 5.4593e-11,\n 1.0404e-10, 2.7232e-10, 7.8472e-11, 3.9605e-13, 3.3948e-11, 9.2276e-14,\n 1.2010e-11, 5.9504e-10, 1.1956e-10, 7.0510e-10, 1.8841e-12, 2.5383e-12,\n 3.0565e-11, 5.5882e-12, 3.9693e-12, 6.5337e-11, 4.1810e-11, 3.8899e-12,\n 1.8343e-13, 1.9600e-12, 1.0628e-12, 4.4045e-12, 3.0882e-11, 4.5432e-12,\n 1.4251e-10, 4.2240e-10, 9.3437e-12, 1.2205e-12, 3.8144e-12, 3.0268e-12,\n 5.0865e-10, 1.2964e-11, 7.6336e-12, 3.8727e-12, 4.4613e-12, 3.5508e-11,\n 1.2287e-12, 8.1562e-11, 2.8665e-11, 5.5285e-10], device='cuda:0')" + }, + "50": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 1.3142e-17, 4.8002e-18, -5.6052e-45, ..., -1.9261e-16,\n 2.6012e-15, 5.6052e-45],\n [-4.8376e-17, 2.7395e-17, -5.6052e-45, ..., -4.4080e-15,\n -5.7500e-14, -5.6052e-45],\n [ 2.5589e-17, -4.5538e-17, -5.6052e-45, ..., 3.4269e-16,\n 7.0149e-15, 5.6052e-45],\n ...,\n [-3.7568e-18, 3.0538e-17, 5.6052e-45, ..., 1.1394e-15,\n -1.8510e-15, -5.6052e-45],\n [-2.1861e-17, 9.8608e-18, 5.6052e-45, ..., -4.1656e-17,\n -1.3168e-15, 5.6052e-45],\n [-6.2673e-17, -9.5321e-17, 5.6052e-45, ..., -1.8787e-16,\n -1.6627e-14, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.2180e-11, 1.1524e-11, 4.7956e-16, ..., 1.2561e-11, 4.9598e-12,\n 3.8493e-13],\n [4.9645e-13, 3.6314e-13, 2.8164e-17, ..., 4.9666e-13, 4.0888e-13,\n 1.5150e-14],\n [1.0990e-13, 2.9719e-13, 9.2870e-19, ..., 2.3184e-13, 3.0612e-14,\n 2.2115e-15],\n ...,\n [1.4562e-12, 1.1405e-12, 1.6048e-15, ..., 2.6429e-12, 7.7852e-13,\n 2.8239e-13],\n [2.9993e-13, 2.3694e-13, 1.7174e-16, ..., 4.7312e-13, 1.2703e-13,\n 4.8143e-15],\n [5.8203e-13, 3.0275e-13, 9.7244e-17, ..., 1.4917e-12, 3.0303e-13,\n 3.1104e-15]], device='cuda:0')" + }, + "51": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 1.6155e-13, -3.7633e-12, 4.8841e-13, 4.0333e-14, -4.1415e-13,\n 5.5017e-13, -3.0774e-15, -1.8883e-13, 4.0810e-13, 9.9650e-14,\n 2.1392e-13, 8.6085e-13, 1.1643e-13, 2.3281e-13, 1.2178e-13,\n 1.0186e-13, -5.6880e-13, 3.9334e-13, 9.1626e-13, -1.0648e-12,\n 1.3493e-13, -9.6806e-14, 4.7418e-13, 6.2806e-15, -1.0763e-13,\n -4.2839e-13, 7.4098e-14, 6.0520e-13, -3.5437e-13, 4.4107e-13,\n -3.7814e-13, -2.4595e-13, -8.8573e-14, 1.0154e-13, 2.8386e-14,\n 7.5582e-14, 5.4992e-14, 1.1747e-13, -3.8559e-13, 3.2721e-13,\n 1.0042e-12, -3.2544e-14, 6.3982e-13, -7.7053e-13, -1.0498e-12,\n 1.3727e-13, -2.3361e-13, -4.9907e-13, 9.4760e-13, -6.0317e-13,\n 2.1026e-13, 2.2774e-13, 8.7437e-13, 2.6578e-13, 7.9208e-14,\n 3.0716e-13, -1.0164e-12, 1.0016e-13, 6.0244e-13, -6.5290e-13,\n -2.9079e-13, 1.0237e-13, 3.6901e-13, 6.2292e-13, -1.0636e-12,\n 2.3029e-14, -5.4365e-13, 9.3479e-13, -4.7431e-13, 4.1504e-13,\n -2.0553e-13, -6.2262e-13, -3.1116e-14, -2.1998e-13, -1.5402e-14,\n 7.3673e-14, -9.8082e-13, 6.8694e-13, -2.3765e-12, 4.0407e-14,\n 4.1038e-13, -1.7281e-13, 2.4353e-12, -1.3223e-14, 4.9480e-13,\n 2.2007e-13, 5.4968e-13, -3.2984e-12, -7.2484e-13, -3.1462e-13,\n 3.2446e-13, -2.7850e-13, -1.3127e-12, -1.4362e-13, -7.4727e-14,\n -5.8778e-13, -4.6704e-13, -6.5605e-14, 1.5623e-13, 7.3266e-13,\n -3.5877e-13, 7.1067e-13, 2.6529e-13, 7.6396e-13, -2.7136e-13,\n 2.1782e-13, 6.3161e-13, 7.5514e-13, -1.3894e-13, -1.9195e-13,\n -1.5469e-12, 9.7631e-14, -1.5415e-13, 1.7369e-13, -1.3268e-13,\n 2.3344e-13, 8.5686e-13, 9.7714e-13, -9.4666e-13, -9.8910e-13,\n 8.4151e-14, 5.3272e-13, -2.8177e-13, 2.0385e-13, 8.7333e-14,\n -3.0971e-13, -2.6731e-12, 4.3804e-13, -1.0101e-13, 2.8463e-14,\n -5.6502e-13, 8.7286e-14, 6.0364e-13, 1.1224e-13, 2.1312e-13,\n -6.1185e-13, -7.2378e-13, 5.8897e-13, 2.2461e-13, 2.8738e-13,\n -1.0694e-13, -2.9109e-13, 1.2622e-12, 7.6493e-13, 2.2413e-13,\n 6.3120e-14, -4.5249e-13, 8.5969e-13, -4.4662e-13, -2.2966e-13,\n 1.1306e-12, 1.5680e-15, 2.6721e-13, 1.1190e-12, 1.1294e-12,\n 5.7719e-13, 1.1667e-13, 4.2081e-13, 7.3599e-14, 4.1761e-13,\n 4.3809e-13, 3.3127e-14, -2.0660e-12, 2.6612e-13, -1.7323e-13,\n 8.2965e-13, 2.7483e-13, 3.7514e-13, 5.6138e-13, 5.4688e-13,\n 3.6569e-13, 1.7200e-13, -5.1211e-14, 2.0623e-13, -1.1921e-12,\n 1.0164e-13, 4.8808e-13, 1.8841e-12, 1.8283e-14, 1.8016e-13,\n -3.4859e-13, 3.5557e-13, -4.3988e-13, -6.5567e-13, 8.7573e-13,\n -1.1260e-13, -3.9748e-14, -5.3253e-14, -4.0471e-13, 2.9779e-13,\n 5.2531e-13, -1.7241e-13, 1.6419e-13, 1.9877e-13, 4.4861e-13,\n -3.5349e-13, 1.1236e-12, -2.8336e-13, 3.1807e-13, 3.6021e-13,\n -3.5430e-13, -1.0986e-12, 6.0650e-14, -3.1110e-13, -5.8683e-13,\n 1.4225e-13, -9.2558e-13, -4.8644e-13, 5.9247e-13, -2.7142e-13,\n 3.3277e-13, 4.5400e-13, 4.1407e-13, 1.0558e-13, 6.3265e-13,\n 1.6821e-12, 4.9959e-14, 6.3655e-13, 1.5163e-12, -2.8228e-13,\n -7.4901e-13, 8.0877e-13, -2.8000e-14, 1.5612e-13, 9.8045e-14,\n 1.3643e-13, 6.3032e-13, 6.0814e-14, -6.0739e-14, -3.5581e-13,\n 1.1299e-12, 5.9386e-13, 7.1512e-13, -1.0456e-12, -3.0172e-13,\n 3.5767e-13, 3.3050e-13, 1.3685e-12, -4.4299e-14, 5.7139e-13,\n -4.3855e-13, 2.4903e-13, 2.9994e-13, -3.6413e-13, -9.1779e-14,\n 1.0235e-13, 9.2242e-14, 8.0829e-13, -2.9150e-13, -1.0718e-13,\n 3.1954e-14, -7.5209e-14, -3.7239e-13, 2.4026e-13, -9.9115e-14,\n -1.7822e-13, 2.3127e-13, 2.7518e-13, -4.2453e-13, -1.6490e-13,\n -3.2148e-13, 1.2870e-13, 3.4874e-13, 2.7379e-15, 6.6779e-14,\n 2.8652e-13, 9.6722e-13, 1.7340e-13, 5.1606e-13, -9.0253e-13,\n 3.5216e-13, 5.4193e-13, 2.1564e-13, -8.4719e-13, 2.5131e-13,\n -1.4342e-12, 1.4271e-12, 1.8584e-13, 6.7126e-13, -5.0064e-13,\n 1.1505e-13, 1.0649e-12, 2.4055e-14, -9.5449e-15, 3.2559e-13,\n -3.5644e-12, 9.5029e-13, -1.9583e-13, 3.0019e-13, 1.3046e-12,\n -1.5346e-14, 2.5380e-13, -3.3564e-13, 6.7678e-14, 3.7390e-14,\n -7.1882e-13, -3.9723e-13, 5.7494e-13, 1.9925e-13, -6.4854e-14,\n 8.1401e-13, -7.8581e-13, 5.7837e-13, 1.5014e-13, -9.8731e-13,\n 3.8143e-13, 8.2219e-14, -8.7707e-14, 1.9558e-13, 2.2389e-13,\n 2.5154e-13, 8.9141e-14, -1.7456e-12, 2.0009e-12, 1.3381e-13,\n -4.8451e-13, 4.5065e-13, -1.8928e-13, -8.5728e-13, 7.1842e-13,\n 2.1437e-13, 3.3186e-13, 1.8178e-13, 3.4442e-13, -2.4180e-13,\n -2.3575e-13, 5.7029e-13, 3.3558e-13, -9.4924e-15, -5.0390e-13,\n -2.6485e-13, 4.6136e-13, -5.6822e-13, 2.5565e-13, 2.2007e-13,\n 2.0327e-12, 2.6923e-13, 5.0595e-14, -2.2560e-12, 5.2863e-13,\n 9.3557e-13, 4.8173e-13, -2.6076e-13, 1.3400e-13, 4.6822e-14,\n -1.6429e-12, 1.2616e-12, 7.6974e-13, -1.7816e-12, 2.8373e-13,\n 7.6553e-13, -1.8101e-13, 6.8853e-13, 9.0366e-15, -5.6841e-14,\n -9.4425e-13, 4.3194e-14, 2.1785e-12, 1.3604e-13, 2.4471e-13,\n 2.0294e-13, -1.7259e-12, 4.7526e-13, 2.3007e-13, 1.8413e-13,\n 2.0753e-13, 1.2359e-12, 8.1503e-13, -7.7755e-14, -2.4159e-12,\n 1.1949e-12, 8.8246e-13, 2.5820e-13, 1.2708e-12, -7.9294e-13,\n -1.7588e-13, -1.8351e-14, 2.2524e-13, -3.7400e-14, 2.7227e-13,\n -3.1724e-13, 2.5874e-13, 6.9903e-13, 3.3319e-13, -6.0525e-13,\n 3.0470e-13, -1.3880e-12, 3.9000e-13, 1.4770e-13, -8.4222e-13,\n 4.3875e-13, 3.4784e-13, 8.6802e-13, -6.3744e-14, 1.4567e-13,\n 2.3466e-14, -5.6310e-13, -1.1949e-12, 1.8971e-13, -5.6259e-13,\n -1.2328e-13, 1.7273e-13, -6.6825e-13, 2.0916e-13, -6.4288e-13,\n -4.6420e-14, 1.1461e-12, 1.2725e-13, -1.5767e-13, 1.0294e-13,\n -3.4362e-13, -1.3419e-13, -2.9450e-12, 3.8465e-13, -1.3841e-13,\n -4.3424e-14, 4.7039e-13, 1.5662e-12, -9.9239e-13, 1.2300e-13,\n 9.1940e-14, 7.2560e-14, -1.1712e-12, -9.4829e-13, -2.7374e-13,\n 4.4347e-13, 1.4863e-14, 2.2610e-13, 1.2844e-13, 9.2467e-14,\n 1.0954e-13, 4.4100e-13, -6.5367e-13, 8.4893e-14, 7.8251e-14,\n -1.5867e-12, -8.5398e-13, -2.1055e-12, 2.2562e-13, -1.6496e-13,\n 4.0901e-14, -2.4651e-13, 1.1037e-12, 3.0346e-13, 2.0493e-14,\n 1.5463e-13, 9.5840e-16, 1.3474e-14, -6.1697e-14, -1.2675e-12,\n 1.2395e-13, -1.6192e-12, 1.4954e-13, -2.0959e-12, 8.2474e-13,\n -1.1171e-13, -3.9958e-13, 1.2417e-13, 4.8037e-13, 2.4882e-13,\n 1.2702e-13, 3.3029e-13, -1.0392e-12, -9.3004e-13, -1.9444e-12,\n -2.2248e-13, -2.5824e-13, -8.9700e-13, 9.6550e-14, 2.5435e-13,\n 2.8614e-13, 1.7235e-12, 3.2103e-13, 7.9558e-13, -1.6431e-14,\n -7.8946e-13, 4.6690e-13, 2.2397e-13, 1.1738e-13, -4.5017e-13,\n 2.1129e-13, -3.6997e-13, 1.2483e-12, -3.1683e-13, 7.2875e-13,\n -2.9426e-13, 7.8626e-13, -1.6013e-13, -5.2170e-13, 1.2059e-13,\n 5.5333e-13, -1.3331e-12, -2.3972e-13, 6.4458e-13, -7.2242e-13,\n -8.2118e-13, 3.3802e-13, 2.5358e-13, -1.6113e-13, 3.3657e-14,\n 2.5245e-13, -4.0077e-13, 9.1215e-14, 1.8685e-13, 3.5686e-13,\n -8.2322e-14, 1.6373e-13, -3.8417e-13, 4.1632e-13, -7.7399e-13,\n 2.6049e-13, 1.5181e-13, 6.9288e-14, 1.3705e-13, 9.1515e-14,\n 2.4122e-13, 1.8041e-13, -7.1320e-13, 7.4295e-13, 2.7228e-14,\n -2.4210e-14, -6.6648e-15, -5.8345e-14, 8.9196e-13, -1.7595e-12,\n 3.5962e-13, 2.5361e-13, 7.8102e-15, -1.5059e-12, -8.4759e-13,\n -1.0798e-12, 6.4428e-13, 1.2300e-13, 1.4106e-12, 1.3537e-12,\n 4.4646e-13, 1.4095e-14, 9.0668e-15, 5.4725e-13, -1.5879e-12,\n -1.1410e-13, -3.3110e-13, 2.6683e-13, 1.7418e-13, -2.1531e-12,\n -2.6343e-12, 4.1791e-13, -1.1325e-14, 4.7501e-15, 9.6620e-13,\n -3.1895e-13, 3.2104e-13, 7.5980e-13, -8.9306e-14, 1.2055e-13,\n 2.0399e-13, 5.1245e-13, 1.0488e-13, 1.6474e-13, -2.9564e-13,\n 9.1856e-13, -8.3984e-14, -6.3180e-13, 9.0692e-13, 4.4349e-14,\n 1.2470e-13, -2.5522e-13, -1.4196e-12, -8.4564e-14, -2.2949e-13,\n -2.1133e-14, 1.2951e-13, 1.9252e-13, -3.6221e-13, -3.1024e-13,\n -1.4166e-12, 5.2955e-14, 5.4208e-13, -3.6252e-13, -2.6093e-12,\n -8.7178e-14, 3.3026e-14, -6.9148e-13, -7.1752e-13, -5.9116e-13,\n -1.7146e-13, -6.3571e-13, -8.3028e-13, 6.6971e-13, 9.1727e-13,\n -1.2296e-13, 2.1368e-13, -5.7112e-14, 7.3113e-14, 5.4137e-14,\n 1.2974e-13, 3.4282e-13, -6.1412e-13, 8.7186e-14, 7.1766e-13,\n -1.4789e-12, 1.1286e-13, 5.1155e-13, 2.9850e-13, 1.5928e-13,\n -2.0502e-13, -2.0717e-13, 7.9521e-13, 2.3803e-13, -1.3966e-12,\n 5.1386e-13, 1.6117e-13, 7.2998e-14, -3.4543e-13, 1.0091e-12,\n -2.5404e-13, -4.2861e-13, -8.4582e-14, -9.0499e-13, 3.2324e-13,\n -2.4312e-13, 1.2100e-12, 4.8340e-13, 2.9930e-13, 5.2291e-13,\n -1.8726e-13, 5.6450e-13, -2.9205e-13, -7.0166e-14, -5.1026e-13,\n 4.3944e-13, 4.4700e-13, -6.6374e-14, -4.4481e-13, -7.1120e-14,\n 6.2318e-13, 3.1092e-14, -4.9878e-13, -1.5310e-13, -1.4493e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.8057e-07, 2.5583e-08, 2.4031e-09, 2.2774e-08, 1.2234e-08, 1.6986e-08,\n 8.0290e-07, 1.0956e-06, 3.6785e-07, 3.0673e-09, 1.1474e-06, 7.1655e-08,\n 1.9355e-06, 2.2746e-07, 8.8793e-09, 7.9970e-07, 1.9957e-07, 5.1897e-07,\n 6.0836e-08, 2.3646e-07, 1.5880e-07, 7.8895e-09, 3.7922e-08, 2.6593e-09,\n 1.7523e-06, 1.7368e-07, 1.3776e-08, 1.5333e-06, 6.9596e-08, 2.7623e-07,\n 3.9884e-08, 1.2350e-08, 7.4263e-09, 1.6029e-08, 1.5151e-08, 8.7980e-08,\n 2.0500e-08, 1.6595e-09, 1.9574e-08, 1.3834e-08, 1.0939e-07, 6.0463e-09,\n 2.4656e-07, 7.1307e-08, 7.8986e-08, 8.2524e-08, 2.3129e-09, 1.6677e-09,\n 5.2694e-07, 1.3338e-07, 1.3429e-08, 6.2401e-08, 1.4941e-07, 1.1425e-07,\n 9.6120e-09, 3.0702e-08, 3.8953e-07, 5.7919e-07, 1.3030e-06, 4.4492e-07,\n 1.8501e-08, 1.8579e-09, 4.7364e-07, 4.6085e-09, 3.7395e-07, 3.0559e-08,\n 2.4466e-08, 1.5857e-06, 2.1303e-06, 8.4986e-07, 1.4470e-07, 1.1717e-08,\n 3.4458e-07, 3.6952e-09, 9.5709e-09, 2.5108e-09, 2.7268e-09, 1.1762e-07,\n 2.7026e-07, 8.6159e-07, 3.0494e-07, 2.4035e-08, 3.3204e-07, 3.0109e-07,\n 1.2372e-07, 2.4107e-08, 8.9466e-09, 2.9899e-08, 4.3093e-07, 7.0555e-07,\n 3.5035e-09, 2.7379e-08, 1.2482e-08, 1.3536e-07, 1.5520e-07, 2.5998e-09,\n 1.8905e-08, 2.0266e-06, 2.2736e-07, 2.6327e-06, 2.0140e-08, 3.8676e-07,\n 3.2203e-08, 1.2617e-07, 3.1368e-09, 5.4277e-08, 6.8203e-08, 2.0439e-08,\n 7.8586e-09, 2.4760e-08, 1.1648e-06, 4.9734e-07, 1.8259e-06, 8.8136e-07,\n 1.5077e-07, 6.6084e-09, 5.6612e-09, 8.2412e-07, 4.3762e-08, 2.7718e-07,\n 3.5424e-09, 2.5180e-07, 4.1348e-09, 1.0804e-09, 6.0101e-07, 3.0011e-08,\n 6.5982e-07, 1.1693e-07, 1.8937e-07, 7.6525e-08, 1.9055e-09, 2.1337e-07,\n 1.9322e-07, 6.8299e-07, 6.1454e-07, 2.2638e-06, 9.3780e-09, 2.0150e-09,\n 7.1416e-08, 1.0689e-07, 1.4259e-09, 3.0447e-08, 5.1618e-08, 1.0743e-07,\n 1.9080e-08, 8.4265e-09, 1.9933e-08, 1.1752e-06, 9.7900e-09, 1.6863e-08,\n 5.5618e-08, 7.9182e-08, 3.7200e-07, 8.7756e-09, 2.9133e-07, 7.1046e-09,\n 1.1365e-08, 7.9741e-07, 1.6016e-07, 3.6385e-08, 8.2049e-07, 4.5571e-07,\n 1.5385e-06, 7.4475e-08, 7.2355e-08, 1.3714e-06, 1.2344e-08, 1.6879e-07,\n 3.5534e-07, 8.4717e-07, 2.1766e-08, 1.0351e-06, 6.9857e-08, 4.7778e-09,\n 1.1549e-06, 7.4793e-08, 6.6999e-09, 1.0691e-06, 9.1197e-08, 8.8340e-08,\n 9.4927e-07, 7.3299e-08, 2.1445e-07, 1.7027e-09, 8.9433e-08, 2.2078e-09,\n 1.1672e-06, 1.1955e-07, 4.4463e-08, 2.4934e-09, 1.1607e-07, 3.0330e-09,\n 2.6888e-08, 3.6378e-09, 1.4348e-06, 3.5091e-09, 1.4081e-07, 6.8580e-08,\n 2.7932e-06, 6.4651e-07, 1.7647e-08, 4.1420e-07, 4.7401e-07, 1.1583e-06,\n 2.9903e-09, 1.1299e-09, 1.2023e-06, 4.9574e-09, 8.1173e-08, 7.3251e-07,\n 1.0434e-06, 9.5541e-09, 1.0086e-08, 1.6781e-07, 2.6919e-06, 4.4882e-07,\n 1.4838e-07, 1.8784e-09, 8.2446e-08, 1.1668e-07, 1.5421e-07, 2.8467e-07,\n 1.6139e-06, 9.9509e-09, 1.3243e-09, 1.2788e-08, 4.0753e-06, 7.4617e-09,\n 3.6740e-07, 4.6023e-08, 7.1634e-09, 6.6648e-09, 1.1120e-06, 1.1887e-07,\n 6.2068e-07, 9.4060e-07, 2.0958e-09, 1.2577e-06, 1.9496e-08, 8.5483e-08,\n 2.0130e-09, 5.4113e-07, 9.0578e-07, 8.9877e-08, 7.9423e-09, 1.2482e-08,\n 6.1154e-09, 2.1225e-07, 4.8800e-08, 1.8420e-07, 1.9815e-08, 6.5548e-09,\n 4.5508e-09, 5.2402e-09, 2.5340e-09, 9.4627e-09, 8.2917e-09, 1.6134e-09,\n 8.8842e-08, 3.6036e-07, 7.4903e-07, 2.6898e-09, 4.9552e-08, 2.2937e-09,\n 1.8339e-08, 3.8049e-07, 3.5654e-08, 2.6810e-08, 1.1467e-07, 1.0918e-06,\n 5.0404e-08, 3.4175e-09, 1.8454e-08, 9.6598e-09, 2.1153e-07, 1.3810e-07,\n 1.5611e-08, 2.0331e-09, 4.3847e-07, 3.1210e-07, 2.4462e-08, 4.9475e-07,\n 8.2123e-09, 3.5791e-06, 4.2125e-08, 1.0700e-08, 1.6983e-09, 1.3765e-09,\n 1.1032e-06, 9.0329e-08, 1.1585e-08, 2.4375e-08, 1.2047e-07, 3.6736e-08,\n 8.1545e-09, 7.2125e-09, 8.7651e-07, 2.6711e-07, 1.0855e-07, 8.8426e-08,\n 8.1659e-07, 4.2858e-08, 6.4189e-07, 6.2461e-07, 4.0169e-07, 1.6722e-09,\n 7.9565e-09, 4.8877e-08, 7.7882e-08, 2.2955e-07, 2.2915e-07, 2.0749e-08,\n 4.7545e-07, 1.7153e-07, 9.5012e-07, 1.1817e-09, 3.6616e-09, 2.7595e-07,\n 1.2837e-06, 7.2406e-07, 9.1596e-10, 1.0416e-07, 1.2880e-07, 1.1991e-08,\n 3.4220e-09, 2.6222e-07, 1.8892e-07, 1.3537e-08, 2.6631e-07, 4.8287e-09,\n 7.2555e-07, 1.6182e-08, 6.6137e-07, 1.2726e-07, 9.7917e-07, 2.2486e-08,\n 7.5522e-07, 3.9011e-09, 1.2207e-06, 2.2790e-06, 6.8553e-07, 7.4333e-08,\n 9.7866e-09, 1.0368e-08, 2.3834e-08, 8.6423e-07, 3.0507e-08, 1.3217e-06,\n 1.1019e-07, 2.7159e-07, 2.9806e-07, 4.0612e-09, 1.5662e-09, 1.6703e-09,\n 1.4678e-08, 1.5476e-07, 1.4572e-06, 2.2189e-07, 2.1586e-08, 3.3893e-07,\n 2.5970e-07, 8.1172e-07, 4.0163e-08, 4.2455e-07, 9.9111e-07, 9.2853e-08,\n 1.6965e-07, 2.4027e-07, 2.3580e-08, 4.1633e-07, 2.3135e-06, 9.6857e-09,\n 1.0169e-06, 1.7424e-06, 1.1588e-08, 1.9150e-07, 3.0149e-08, 3.1522e-09,\n 3.4446e-09, 1.3040e-06, 4.2128e-07, 5.1979e-07, 5.2172e-07, 1.9377e-08,\n 1.5685e-09, 5.2494e-07, 8.3606e-09, 6.3538e-09, 1.1530e-06, 5.8961e-07,\n 6.5831e-07, 1.7185e-09, 1.0192e-06, 2.2965e-08, 4.9679e-09, 2.4115e-09,\n 2.0628e-09, 3.9843e-07, 4.4008e-08, 2.0603e-06, 1.0540e-09, 9.2859e-09,\n 3.2166e-06, 5.4184e-07, 1.7438e-07, 1.7305e-08, 3.7409e-09, 2.3505e-09,\n 9.4543e-07, 2.2327e-07, 1.4093e-07, 1.8408e-09, 6.7111e-08, 2.0738e-08,\n 2.7717e-08, 1.1732e-08, 3.8552e-09, 9.1686e-07, 2.9251e-06, 7.5651e-09,\n 1.4028e-08, 4.7002e-09, 1.0985e-06, 3.4677e-07, 1.4771e-06, 2.9096e-07,\n 8.2734e-09, 1.1024e-08, 3.0568e-08, 4.0634e-08, 1.8398e-09, 2.8170e-07,\n 6.8149e-08, 6.7873e-07, 3.8573e-08, 1.5800e-06, 7.5570e-09, 5.7887e-07,\n 1.6527e-09, 3.3277e-09, 9.2942e-09, 2.4600e-08, 9.0823e-07, 1.0286e-08,\n 4.7136e-08, 1.6563e-07, 7.6974e-07, 1.3191e-07, 1.3788e-08, 6.1876e-09,\n 2.4502e-09, 2.5984e-08, 4.8638e-09, 6.1815e-07, 1.2403e-07, 6.9400e-09,\n 6.2453e-07, 4.7666e-09, 8.2107e-07, 3.1444e-09, 1.0666e-07, 1.0715e-07,\n 4.7720e-07, 2.2057e-08, 1.4036e-06, 1.5826e-08, 1.3441e-06, 3.9413e-09,\n 6.1448e-08, 1.9600e-09, 1.1747e-07, 2.2476e-07, 9.3216e-08, 3.1255e-08,\n 1.1256e-07, 5.4604e-07, 6.4929e-09, 8.9704e-09, 8.2406e-09, 2.5425e-07,\n 4.2580e-08, 2.6462e-09, 2.0612e-06, 7.0623e-07, 5.4587e-08, 2.6986e-06,\n 4.6856e-07, 6.0025e-07, 2.2931e-08, 2.5674e-09, 1.7863e-07, 1.3851e-08,\n 4.5275e-09, 9.9176e-08, 2.1104e-06, 1.3216e-07, 7.0183e-09, 2.1832e-08,\n 1.7958e-06, 3.2983e-09, 1.9460e-08, 1.6479e-06, 1.3341e-07, 9.2052e-07,\n 4.0453e-07, 5.9999e-08, 5.4399e-09, 5.7860e-08, 1.5930e-08, 2.6344e-07,\n 1.7357e-09, 3.1125e-09, 5.5757e-08, 6.2110e-07, 5.7379e-08, 2.0852e-08,\n 1.4327e-08, 4.2947e-08, 2.5170e-06, 3.7155e-09, 1.3116e-06, 1.0420e-09,\n 4.1311e-09, 3.2982e-07, 3.6336e-07, 6.0413e-08, 4.1136e-09, 2.1485e-06,\n 6.1019e-07, 8.0146e-07, 1.0843e-06, 1.2910e-06, 1.8075e-07, 2.1163e-07,\n 5.9756e-07, 1.3552e-06, 3.7390e-07, 4.8010e-08, 3.3805e-07, 2.4003e-07,\n 1.3762e-08, 4.7948e-07, 6.9662e-07, 6.3361e-09, 1.1281e-06, 2.3561e-09,\n 5.8775e-08, 3.4427e-08, 5.1686e-08, 2.1850e-08, 3.7705e-08, 6.4575e-07,\n 1.7370e-06, 2.6736e-07, 4.9705e-07, 5.2672e-07, 6.6961e-08, 1.1403e-08,\n 1.0479e-07, 1.4384e-08, 1.2895e-08, 4.3939e-09, 2.0574e-09, 9.9320e-07,\n 2.5126e-08, 1.7373e-08, 1.7952e-07, 2.6799e-07, 1.5141e-07, 1.6041e-06,\n 1.3294e-08, 5.9263e-07, 3.8509e-08, 4.9232e-07, 2.1572e-08, 2.5633e-07,\n 2.1967e-08, 1.9249e-08, 1.2662e-07, 9.8554e-06, 1.8967e-09, 2.3586e-09,\n 2.3565e-09, 5.5695e-09, 4.1025e-09, 1.9875e-08, 1.9407e-08, 6.2763e-08,\n 1.0101e-07, 6.9159e-09, 8.3789e-09, 7.3928e-09, 4.7092e-09, 8.4749e-09,\n 1.7778e-08, 2.0024e-09, 3.3924e-08, 4.4064e-08, 2.4801e-09, 1.8764e-08,\n 3.9129e-07, 5.9220e-08, 8.3053e-08, 1.3610e-09, 2.5382e-07, 4.3653e-09,\n 5.7565e-08, 3.6175e-07, 5.6818e-07, 1.1705e-06, 6.2179e-08, 3.3170e-09,\n 2.5005e-08, 1.8547e-09, 5.0832e-06, 6.7033e-08, 1.1589e-08, 1.3870e-08,\n 3.0843e-08, 1.9139e-06, 2.3991e-09, 3.8396e-08, 3.9507e-08, 3.2810e-08,\n 5.1831e-07, 1.0705e-06, 1.6864e-07, 1.9818e-07, 3.3365e-08, 6.8137e-08,\n 6.6040e-07, 4.7865e-08, 7.0426e-09, 3.1703e-07, 7.1686e-07, 5.1910e-09,\n 3.3331e-07, 5.2750e-08, 7.7779e-09, 2.3307e-08], device='cuda:0')" + }, + "52": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.9537e-16, -3.8875e-13, -3.2129e-15, 7.8316e-16, -5.0829e-15,\n -1.5821e-15, -1.1635e-14, -2.1260e-15, -9.2628e-15, -1.0563e-15,\n -2.9717e-14, 2.6696e-15, -1.1921e-14, -8.3942e-16, 4.8552e-16,\n -2.2846e-14, -4.7744e-14, 1.8322e-16, -8.4041e-16, -7.8434e-15,\n -9.4993e-16, -1.9532e-15, -1.8639e-15, 1.9128e-15, -3.6849e-14,\n -3.5796e-15, -1.2425e-15, 6.6662e-16, 1.1941e-15, 1.2366e-15,\n 5.7296e-15, 3.9064e-15, -1.7071e-15, -1.5714e-14, 1.8173e-15,\n 6.2763e-16, -1.0109e-15, 1.6705e-15, -1.3056e-15, -8.3325e-16,\n 3.1535e-15, 1.2358e-15, -1.6087e-14, -5.5625e-14, -4.2326e-14,\n -1.2970e-15, -3.5639e-15, -2.2483e-15, 5.0226e-15, -1.3690e-15,\n -5.5379e-16, 1.8669e-16, 4.3032e-17, -8.3668e-17, -8.6762e-16,\n 1.1933e-15, -4.7028e-14, -1.0962e-14, 1.5352e-15, -2.6883e-14,\n 1.5881e-15, 9.6299e-16, -3.5589e-16, 1.8879e-17, -1.0410e-14,\n -1.2233e-15, -8.3653e-15, 2.6599e-15, -5.6812e-15, -1.3164e-15,\n 1.1475e-15, 1.4227e-15, -4.5244e-15, 3.3606e-16, 1.7204e-15,\n -5.4235e-15, -4.2652e-16, 5.3829e-16, -1.6531e-14, -3.5337e-14,\n -1.1570e-15, -4.7757e-14, 4.6950e-14, -1.7360e-15, -3.8329e-15,\n 6.3108e-16, -1.2083e-15, -2.4164e-13, -4.6246e-14, -1.4387e-14,\n -1.6988e-16, -1.0953e-15, -3.5078e-14, -7.2346e-15, -1.7807e-14,\n -5.2213e-15, -1.1250e-14, -6.8273e-15, -3.2272e-14, -3.8471e-15,\n -1.3434e-14, -1.3617e-15, 2.5368e-15, -2.0440e-14, -4.5228e-15,\n 1.8532e-15, -5.9505e-16, 5.7475e-16, -8.6931e-16, -3.1527e-14,\n -1.5003e-14, -4.1504e-14, 1.4237e-15, 3.3460e-15, -1.6483e-15,\n 1.6619e-15, -9.9755e-16, 8.4482e-16, -2.3431e-14, -6.8899e-14,\n -5.5429e-15, 9.9442e-17, 1.3729e-15, 2.6153e-16, -4.4314e-14,\n -1.0594e-15, -1.3103e-13, -1.6515e-14, -2.0694e-15, -2.8700e-15,\n -1.4189e-15, -3.5482e-15, 3.2675e-15, -4.7781e-16, 7.5220e-16,\n -1.2598e-14, -2.6873e-14, -2.9597e-15, -1.3034e-16, 1.8056e-15,\n -3.0099e-15, -1.4393e-14, 4.3774e-15, 2.2069e-15, -2.1592e-17,\n -5.9559e-16, -7.6496e-15, 2.6851e-15, -2.1422e-15, 5.3091e-15,\n 6.6590e-15, 1.3505e-15, 3.7899e-16, -2.0267e-14, 8.2666e-15,\n -1.6693e-15, 5.4569e-17, 1.0749e-15, 1.3970e-15, -2.2910e-15,\n -1.0655e-15, -2.1628e-14, -4.9152e-14, -3.7028e-15, -5.6135e-15,\n 7.8391e-17, -1.3652e-16, 1.3929e-15, -1.4558e-15, 1.3021e-16,\n -6.9847e-15, 9.3291e-16, 1.5643e-15, 7.0374e-16, -1.9537e-14,\n 1.5293e-17, 7.2875e-18, 2.4231e-14, -1.1370e-14, -7.2345e-15,\n -5.0850e-15, -8.3346e-16, -1.0738e-14, 1.9646e-15, -4.2565e-14,\n 6.2546e-16, -1.2036e-14, 1.2246e-15, -2.4115e-14, 1.7745e-15,\n 2.4420e-16, -1.7896e-14, -1.5713e-15, -4.6802e-17, 1.7196e-15,\n -4.2068e-15, 7.1648e-15, -1.0383e-14, -1.0173e-15, -6.8764e-16,\n -1.8728e-14, -9.5218e-14, 6.3248e-18, -3.9302e-15, -1.1734e-16,\n 5.9049e-16, -3.0466e-14, 9.9634e-16, 6.5075e-16, -2.5333e-14,\n 7.2765e-18, -6.0527e-16, -1.0150e-14, 4.8642e-16, -3.3020e-15,\n 1.2785e-14, -1.2367e-15, 1.3103e-16, 1.3337e-14, 6.5325e-15,\n -1.6358e-14, 8.6697e-16, -1.9441e-14, -5.7920e-17, 8.7003e-16,\n -6.6098e-15, -6.5317e-15, -5.6567e-15, -4.5615e-15, -4.1178e-14,\n -1.0858e-14, 1.7058e-15, 6.3246e-15, -2.8983e-14, -5.4202e-16,\n -5.4019e-16, -1.9453e-15, 1.7107e-14, 3.3634e-15, 1.8783e-16,\n -8.1205e-16, -8.8943e-15, -5.3167e-16, -3.1177e-14, 3.2687e-15,\n 3.3920e-16, -7.0525e-16, 3.2308e-15, 3.9907e-15, 1.1119e-15,\n -1.8254e-15, 8.3126e-16, 8.4028e-16, 3.0617e-16, -3.6412e-15,\n -1.3911e-14, 5.0934e-16, -3.6605e-15, -1.6600e-14, -2.9109e-14,\n -1.5752e-15, 8.7552e-16, 8.1943e-16, 3.6495e-15, -1.1465e-16,\n -4.4510e-16, 5.0820e-15, -4.7015e-14, -2.2450e-15, 3.3070e-15,\n 4.1110e-16, -3.6985e-15, -1.9517e-16, -2.1260e-14, -7.6184e-17,\n -4.1626e-14, -5.8856e-14, -2.5495e-16, -1.9980e-16, -1.4268e-14,\n 1.7256e-15, 2.1152e-15, -4.7015e-15, -1.6278e-14, 4.6302e-16,\n -6.3023e-14, 6.3747e-15, 2.2993e-15, 3.1919e-16, 3.9077e-15,\n -4.4972e-15, -2.1711e-17, 1.3993e-15, -1.0952e-15, -1.2863e-15,\n -1.8405e-14, -1.2579e-14, 7.0210e-16, -4.6596e-16, -1.1933e-15,\n 3.4055e-15, -1.5416e-14, -2.4461e-15, -5.2394e-15, -1.0089e-14,\n 3.7976e-17, -3.1821e-14, -9.6526e-16, -5.1380e-15, -1.2197e-14,\n 1.2420e-15, 5.3349e-16, -1.2673e-14, 1.1900e-14, -1.8911e-15,\n -3.2142e-15, -7.6927e-16, -5.8466e-16, -5.1388e-14, -2.3742e-16,\n -7.2574e-16, 2.1702e-18, -1.6585e-14, -1.9547e-15, -5.4663e-15,\n -3.1455e-14, -5.1833e-16, -2.1613e-14, -6.4030e-15, -4.8089e-15,\n -1.5026e-14, -1.4492e-15, -1.2539e-14, 1.0477e-15, 9.7557e-16,\n 3.5145e-14, 1.9540e-16, 9.5824e-16, -4.5958e-14, -3.2709e-14,\n 2.2085e-15, -8.8798e-16, 6.9780e-17, -4.3619e-15, -1.3945e-15,\n -5.1720e-14, 5.1635e-15, -7.0999e-15, -2.5887e-13, 2.2008e-15,\n -6.2037e-15, -2.4258e-15, 2.9418e-15, 2.5022e-15, 3.1550e-15,\n -3.1364e-14, -3.2656e-15, 3.3669e-14, -4.4684e-16, -9.7739e-15,\n 4.0444e-16, -3.4938e-14, -2.0841e-15, -9.8316e-16, -1.8476e-18,\n 2.2527e-16, -1.9256e-14, 1.4275e-15, -3.9844e-16, -8.6215e-14,\n 1.0156e-14, 5.5463e-15, -4.7842e-14, 1.1048e-14, -8.1943e-14,\n -1.6483e-14, -6.2566e-15, -1.0046e-16, -6.9638e-15, 6.3968e-16,\n -2.6163e-14, -3.8316e-15, -5.5394e-16, -3.2086e-15, -3.0517e-15,\n -1.2355e-14, -7.2113e-14, -9.2921e-16, 1.6937e-16, -5.1288e-14,\n -1.9127e-15, -1.1479e-16, 3.2546e-15, 1.9370e-15, 9.4626e-16,\n -7.6225e-16, -3.7552e-15, -5.9594e-15, -4.1896e-15, -8.2796e-15,\n -5.3157e-15, -1.3840e-15, -2.1001e-14, -8.7507e-17, -1.0835e-14,\n -2.1571e-15, 1.3701e-15, -1.0483e-14, -1.3217e-14, -7.5447e-15,\n -1.1378e-14, -9.5013e-15, -5.8670e-14, -3.5607e-16, -4.3330e-15,\n -8.8333e-15, -6.8909e-15, 8.2729e-15, -3.4636e-14, -1.2094e-15,\n -5.0152e-15, 1.6750e-15, -4.0512e-14, -4.0220e-14, -1.4339e-14,\n 5.1762e-16, -6.6580e-14, 6.3695e-16, 6.2512e-16, 3.7737e-16,\n -2.8116e-16, -2.3591e-16, -2.9077e-15, -6.1280e-15, 7.8938e-16,\n -2.2576e-14, -7.5815e-18, -3.6595e-14, -7.0457e-16, 1.5254e-15,\n -4.6149e-15, 5.2705e-16, 1.4907e-14, -1.1367e-15, 1.2269e-15,\n -4.8937e-16, -4.6185e-15, -3.6708e-16, -3.1646e-15, -1.9544e-14,\n -4.7223e-16, -8.0418e-14, -7.5688e-15, -4.0639e-14, -1.3918e-15,\n -1.3903e-16, -3.7268e-14, -4.7712e-17, -1.8139e-14, -3.3222e-16,\n 1.0811e-16, -2.8287e-15, -7.5594e-14, -5.0998e-14, -4.4700e-14,\n 4.3071e-15, -1.6520e-14, -4.5673e-15, -1.5853e-15, 6.6322e-17,\n -1.4244e-16, 1.6136e-14, -1.4538e-14, -2.7125e-14, 2.3145e-15,\n -9.0348e-15, -1.1622e-14, -6.9595e-16, 2.7067e-16, -7.5310e-15,\n 2.4333e-16, -8.4376e-15, 1.0437e-14, -9.0207e-15, 1.0302e-15,\n -2.4132e-14, 2.6766e-15, -2.4169e-14, -2.7607e-14, -5.3791e-15,\n -1.8462e-16, -1.2598e-14, -8.3781e-15, 5.1599e-16, -1.0140e-13,\n -2.0553e-14, -5.0892e-16, -6.5115e-16, -2.0265e-15, -7.2511e-16,\n 3.5874e-16, -3.6927e-15, -1.2232e-15, 5.0220e-16, 1.5789e-16,\n -1.0962e-14, 7.5460e-16, -8.6708e-15, 1.9030e-16, -1.5892e-14,\n -5.4434e-16, 2.0425e-16, 1.0499e-15, -3.4174e-15, 1.0417e-16,\n 1.1785e-15, -9.8640e-15, -1.7659e-14, 1.0411e-15, -6.0684e-17,\n 2.0413e-15, 1.5597e-15, -4.8385e-16, 1.1560e-15, -3.0127e-14,\n -6.0760e-16, 4.7610e-17, -3.2893e-14, -1.0084e-13, -1.5522e-14,\n -5.0181e-14, 2.8848e-15, -1.4089e-16, 4.4735e-15, 8.1055e-15,\n -1.8346e-15, -1.1824e-14, 5.3767e-16, -9.4068e-15, -1.2800e-13,\n 5.7074e-16, -4.8508e-14, -5.0385e-15, 3.6286e-16, -4.7618e-14,\n -7.3935e-14, -1.3968e-14, 1.3451e-15, 2.8588e-15, -5.7539e-15,\n 4.4052e-15, -9.5168e-17, 3.4861e-16, -8.6927e-15, 1.1170e-15,\n -4.6840e-16, -2.0150e-16, -5.2875e-15, -2.0512e-16, -1.0943e-15,\n 4.2304e-15, -1.3771e-14, -5.5455e-16, 2.8912e-15, 2.5494e-16,\n 9.7462e-16, -1.4439e-14, -2.1630e-14, -2.4018e-15, -3.2341e-15,\n -1.5991e-15, -2.4364e-15, 1.8176e-16, -1.1337e-14, -8.9335e-15,\n -1.3576e-14, 2.8220e-15, -4.1114e-16, -1.8230e-14, -9.9846e-14,\n -1.0498e-14, -1.7609e-15, 1.7881e-15, 1.5356e-15, -4.5295e-15,\n -3.4862e-15, -4.4619e-14, -5.4428e-14, 5.8276e-16, 4.3624e-15,\n 2.1434e-15, -1.9639e-15, 1.0164e-15, 2.1921e-15, -1.6277e-15,\n -1.3818e-14, 5.4417e-16, -3.0894e-14, 5.0923e-16, -4.7696e-17,\n -3.8075e-14, -1.2784e-16, -3.6136e-14, -7.1857e-16, 2.4368e-15,\n -1.2226e-14, -2.9373e-15, 2.6120e-15, -7.7291e-15, -3.6438e-14,\n -6.2498e-16, 1.9466e-17, -9.8609e-16, -3.3199e-15, 4.7675e-15,\n -7.7873e-14, -1.3299e-14, 1.2008e-16, -2.4125e-14, -3.0932e-16,\n 1.1541e-15, 1.2217e-14, -7.7638e-14, 1.9562e-15, -1.4656e-14,\n 3.9650e-15, 2.2083e-15, -3.0567e-15, 1.2249e-15, -1.2734e-15,\n -7.6017e-16, -1.2377e-15, -9.6827e-16, -1.7259e-14, 6.7859e-16,\n 1.1825e-15, 3.1434e-17, -5.9984e-14, -3.0858e-14, -1.1262e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.0261e-12, 6.1271e-12, 4.1739e-14, 2.7638e-13, 4.1881e-13, 1.2247e-12,\n 2.2047e-11, 4.8575e-11, 3.7174e-10, 2.3513e-13, 1.4851e-09, 8.4433e-13,\n 6.6569e-10, 1.2189e-11, 5.3516e-13, 3.8964e-10, 3.4858e-12, 8.8167e-12,\n 4.6549e-13, 1.6585e-11, 1.3434e-10, 4.0085e-14, 1.4150e-12, 7.6384e-14,\n 7.3394e-10, 5.2196e-12, 1.9674e-12, 3.7445e-10, 5.7913e-13, 6.0727e-11,\n 1.7369e-11, 4.0087e-12, 2.1724e-12, 1.2039e-11, 3.5904e-12, 1.2346e-11,\n 4.2292e-13, 4.8307e-14, 3.1887e-12, 1.3170e-13, 5.2733e-13, 1.3299e-13,\n 2.3578e-11, 4.0826e-12, 3.7497e-12, 8.0495e-13, 6.6310e-13, 2.9429e-13,\n 2.3870e-10, 2.3654e-12, 5.3149e-13, 5.5020e-13, 7.3666e-12, 1.7982e-12,\n 1.4735e-13, 5.5144e-13, 3.1824e-11, 1.1055e-10, 1.8190e-10, 1.7060e-10,\n 1.1656e-13, 3.5131e-14, 1.9285e-11, 5.3733e-13, 5.9427e-12, 1.9876e-11,\n 2.9338e-13, 7.0855e-11, 3.8057e-10, 5.2030e-11, 7.4562e-13, 4.0197e-12,\n 8.2807e-11, 4.3652e-14, 1.4471e-12, 3.3837e-12, 2.4631e-12, 3.4763e-12,\n 3.5859e-11, 4.3540e-10, 5.8966e-11, 2.1306e-12, 3.0659e-10, 3.1328e-11,\n 6.3166e-13, 6.2062e-12, 1.5592e-13, 5.5509e-12, 3.7519e-11, 2.7198e-10,\n 1.1708e-13, 5.7391e-12, 4.1762e-13, 7.1241e-13, 5.1906e-12, 2.4209e-12,\n 1.9519e-13, 1.5397e-10, 1.4345e-10, 7.4042e-10, 1.3240e-12, 1.1842e-11,\n 1.4523e-12, 2.1107e-11, 3.5612e-12, 7.4796e-13, 9.8103e-12, 1.5544e-12,\n 8.7108e-14, 1.1128e-12, 2.4972e-10, 3.1194e-10, 9.7357e-11, 2.1904e-10,\n 2.1247e-11, 8.9735e-14, 7.9074e-13, 6.2291e-12, 4.8705e-12, 1.1421e-10,\n 3.8373e-12, 2.1404e-12, 4.8342e-13, 1.2580e-12, 5.0150e-10, 3.3533e-13,\n 1.3995e-10, 7.8710e-11, 3.8555e-11, 7.6666e-12, 2.8993e-13, 3.0317e-12,\n 3.9313e-12, 2.6319e-10, 4.5238e-12, 4.5241e-10, 2.4167e-13, 1.7214e-12,\n 2.7721e-13, 1.1325e-12, 1.0850e-12, 4.0119e-12, 1.3063e-12, 1.5367e-12,\n 9.2515e-13, 2.2276e-12, 2.4611e-12, 3.4394e-11, 1.0126e-11, 7.0491e-12,\n 9.2813e-12, 9.5971e-13, 3.0956e-12, 5.3028e-13, 2.0608e-10, 1.1719e-12,\n 5.1400e-13, 2.5732e-10, 1.3694e-11, 1.0554e-11, 4.3663e-11, 1.8281e-10,\n 4.5910e-10, 2.5019e-13, 4.0206e-13, 5.9020e-10, 2.6549e-12, 9.0384e-13,\n 1.5501e-10, 1.4353e-10, 2.6198e-13, 1.2920e-11, 1.2378e-12, 5.1894e-14,\n 5.3376e-11, 1.2949e-12, 1.0945e-13, 9.0451e-10, 5.5123e-11, 8.9724e-13,\n 2.5604e-10, 2.4397e-12, 2.2275e-12, 6.9605e-13, 3.1396e-11, 3.4501e-13,\n 3.1857e-10, 4.7865e-13, 1.3722e-12, 3.5112e-14, 1.2026e-12, 1.0937e-13,\n 7.0636e-12, 3.8896e-13, 6.4226e-11, 3.0870e-12, 1.1608e-10, 8.2048e-13,\n 1.2921e-09, 1.1151e-11, 1.6148e-12, 1.7916e-10, 1.1670e-11, 1.1695e-10,\n 4.3158e-12, 6.1830e-14, 5.0622e-11, 5.6167e-13, 3.9750e-13, 1.5987e-11,\n 9.3992e-11, 1.5662e-13, 1.1267e-12, 2.0425e-12, 8.4500e-10, 1.3448e-11,\n 9.6043e-12, 2.4493e-12, 1.0050e-11, 1.5085e-12, 2.2893e-11, 5.5128e-11,\n 1.7076e-10, 8.6229e-13, 1.5595e-12, 2.9764e-13, 1.2860e-09, 7.1178e-12,\n 1.4294e-10, 1.4462e-11, 2.3036e-13, 8.9167e-14, 1.5151e-10, 1.3022e-11,\n 1.4800e-11, 4.9032e-11, 1.8395e-13, 3.8246e-10, 3.4652e-13, 2.0498e-11,\n 3.6398e-13, 1.2416e-10, 1.1835e-11, 1.9458e-11, 3.8763e-14, 1.5447e-13,\n 1.0295e-13, 1.4256e-12, 1.1988e-11, 1.4048e-11, 5.4146e-12, 1.1327e-12,\n 2.1109e-13, 1.7825e-12, 6.0105e-13, 2.6218e-13, 7.5337e-12, 5.5304e-14,\n 1.2909e-11, 1.0070e-10, 1.2286e-10, 3.0572e-14, 2.4302e-11, 9.2834e-14,\n 5.4243e-12, 7.0115e-12, 4.6253e-13, 3.1532e-12, 1.2512e-12, 4.1476e-11,\n 2.9377e-12, 6.8523e-14, 1.2459e-13, 1.5697e-13, 4.1946e-12, 3.6217e-12,\n 5.9348e-12, 2.5873e-14, 1.3949e-11, 2.0592e-10, 3.0880e-13, 8.9164e-12,\n 2.7856e-13, 1.2659e-09, 2.0579e-11, 2.0704e-13, 2.2377e-14, 8.0140e-13,\n 3.6503e-11, 3.3734e-11, 7.3910e-12, 6.2120e-13, 4.7208e-12, 1.2514e-11,\n 1.5177e-12, 1.3230e-13, 3.0808e-11, 5.5904e-12, 6.1864e-11, 4.1520e-13,\n 1.4628e-10, 3.7231e-12, 5.2553e-12, 7.8009e-12, 2.0120e-10, 6.2848e-14,\n 5.3642e-13, 1.3041e-12, 9.9363e-13, 4.3205e-11, 3.0445e-11, 6.2634e-12,\n 6.7504e-12, 2.4800e-12, 1.0142e-10, 4.6287e-13, 2.7589e-13, 4.1395e-12,\n 7.4372e-10, 4.6989e-11, 5.6977e-13, 4.0147e-12, 5.6909e-11, 2.5711e-12,\n 3.1671e-12, 2.4738e-11, 1.9695e-12, 1.3572e-12, 5.7352e-11, 6.2226e-12,\n 2.0726e-10, 2.6524e-12, 5.2324e-11, 1.4813e-10, 4.0644e-10, 6.5714e-12,\n 3.4980e-11, 3.4615e-13, 5.7967e-11, 1.0228e-09, 4.1465e-11, 4.3509e-12,\n 1.1418e-13, 7.4826e-12, 4.6766e-13, 9.3275e-11, 3.5300e-13, 2.8477e-10,\n 1.1563e-11, 1.2602e-12, 1.1598e-10, 5.5247e-13, 8.5375e-14, 2.8826e-13,\n 1.8978e-13, 4.1342e-11, 1.8353e-10, 9.7367e-12, 7.3564e-12, 1.9220e-11,\n 3.2053e-12, 2.9598e-10, 6.4132e-13, 2.2552e-10, 2.0454e-10, 3.5102e-13,\n 3.7352e-11, 1.0888e-10, 5.3281e-13, 6.3904e-12, 1.6933e-09, 1.1821e-13,\n 6.8045e-10, 7.4547e-10, 1.9364e-12, 3.6962e-11, 8.3904e-12, 1.2547e-13,\n 9.9824e-13, 4.5161e-11, 4.3978e-11, 1.6058e-10, 1.4691e-10, 1.0021e-13,\n 2.0770e-13, 1.3680e-11, 5.7208e-13, 1.0332e-12, 1.8476e-10, 3.6373e-10,\n 1.6328e-10, 3.2064e-14, 4.1100e-11, 7.9604e-12, 4.2007e-13, 8.3841e-14,\n 1.8493e-12, 4.9680e-11, 1.6976e-11, 3.5813e-10, 3.9160e-13, 1.7077e-12,\n 6.3739e-10, 2.3166e-12, 7.4752e-13, 2.5154e-13, 2.1493e-14, 2.1255e-13,\n 1.9351e-11, 9.5019e-12, 1.7754e-11, 1.0767e-14, 9.6016e-12, 1.8841e-12,\n 8.9957e-12, 9.1030e-12, 3.3444e-13, 1.3455e-10, 4.1840e-10, 2.5113e-13,\n 1.0399e-13, 7.7913e-13, 1.8240e-10, 1.2922e-10, 4.8381e-10, 1.2372e-11,\n 1.6384e-12, 1.5993e-13, 9.8768e-14, 3.8748e-13, 8.5593e-14, 2.4455e-12,\n 1.6853e-12, 1.5794e-10, 5.3912e-13, 2.1883e-10, 1.9295e-12, 1.0170e-10,\n 5.7363e-13, 2.2188e-14, 9.3301e-13, 6.6948e-12, 6.9270e-10, 4.0434e-13,\n 1.7296e-12, 2.2847e-11, 3.6888e-11, 1.2010e-12, 3.8607e-12, 1.4809e-12,\n 7.2241e-14, 1.0619e-11, 6.3822e-13, 7.0199e-11, 2.9885e-12, 1.3881e-12,\n 1.0512e-10, 2.3221e-13, 4.8081e-10, 2.1991e-13, 1.3316e-12, 1.9345e-11,\n 1.1074e-10, 3.3028e-13, 5.2438e-10, 4.5653e-13, 2.4124e-10, 1.1632e-13,\n 1.7314e-11, 5.0701e-14, 1.2908e-12, 4.9954e-12, 8.2392e-12, 1.3600e-12,\n 6.4291e-13, 1.6251e-11, 2.8146e-13, 5.4460e-14, 1.5429e-13, 3.0440e-12,\n 6.4936e-13, 1.9066e-13, 1.2079e-09, 1.1250e-10, 4.6945e-11, 8.6623e-10,\n 3.3249e-10, 1.1076e-10, 2.4782e-13, 1.6990e-12, 1.9050e-12, 1.3559e-12,\n 2.0016e-13, 3.3123e-11, 1.6236e-09, 4.3921e-13, 1.8361e-13, 2.8525e-13,\n 3.5696e-10, 5.7179e-14, 6.9261e-14, 1.3128e-10, 2.3245e-11, 5.8364e-11,\n 2.3069e-11, 5.6758e-13, 4.7463e-14, 1.4137e-12, 3.6019e-12, 1.3225e-10,\n 1.3570e-13, 3.5203e-13, 2.4451e-11, 1.2974e-11, 1.8796e-11, 8.2379e-13,\n 7.0266e-13, 4.4906e-12, 7.6964e-11, 3.5740e-14, 3.8714e-11, 3.0833e-13,\n 3.3657e-13, 2.9849e-10, 6.4507e-11, 8.2254e-13, 6.6118e-14, 1.7976e-09,\n 6.8351e-10, 1.1996e-10, 9.9340e-11, 1.5553e-11, 2.3154e-12, 6.6097e-11,\n 8.1371e-11, 6.6937e-11, 2.1947e-10, 1.4790e-11, 1.0558e-10, 2.6843e-11,\n 1.1032e-13, 3.3767e-11, 9.6053e-11, 1.6475e-13, 6.0406e-11, 4.8332e-14,\n 1.0965e-11, 1.2425e-11, 6.3262e-12, 2.4477e-13, 1.8602e-12, 8.3268e-11,\n 1.2639e-09, 2.6438e-10, 2.9272e-12, 5.6208e-12, 3.0195e-12, 2.9838e-14,\n 6.2688e-12, 4.2376e-12, 2.1142e-13, 3.8090e-12, 7.0273e-13, 4.6268e-10,\n 5.7007e-12, 7.3499e-13, 4.1854e-11, 1.0999e-12, 6.4086e-12, 8.4748e-11,\n 2.7754e-13, 1.5582e-10, 1.6932e-13, 2.7852e-10, 1.2635e-11, 3.5050e-11,\n 1.5083e-13, 1.0530e-13, 1.9672e-11, 5.5864e-09, 4.3073e-13, 1.6329e-13,\n 2.2742e-12, 8.7889e-14, 4.5347e-12, 6.3890e-12, 9.5932e-12, 3.2993e-12,\n 6.1734e-13, 4.5595e-13, 1.3714e-12, 8.9327e-13, 1.3169e-14, 2.4835e-13,\n 1.7755e-12, 1.0114e-13, 1.7144e-11, 7.2585e-12, 1.2905e-13, 1.3100e-12,\n 4.5811e-11, 4.8000e-12, 1.1188e-11, 2.3309e-13, 2.6056e-12, 1.6670e-13,\n 4.3156e-13, 1.6791e-11, 3.0687e-11, 4.8221e-10, 4.5664e-12, 2.5431e-14,\n 5.8102e-12, 1.5779e-12, 1.1243e-09, 1.3148e-11, 4.0182e-13, 1.0529e-13,\n 1.1611e-13, 2.1041e-10, 1.4790e-13, 3.5083e-12, 7.2808e-12, 8.0440e-12,\n 3.6726e-12, 6.7463e-11, 2.2374e-12, 1.1747e-12, 2.4145e-13, 5.2097e-13,\n 2.4013e-11, 5.2810e-13, 1.2556e-12, 4.4314e-11, 9.0511e-11, 5.3324e-14,\n 9.2771e-11, 1.8359e-11, 5.3718e-13, 1.0036e-12], device='cuda:0')" + }, + "53": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-6.3848e-15, -1.2565e-13, 5.8497e-15, -6.0621e-16, -1.4650e-14,\n -4.0816e-15, -1.5174e-14, -1.6016e-14, -8.4173e-15, 1.1024e-15,\n -1.9382e-14, 9.2175e-15, -1.5300e-14, -2.0955e-15, -6.9351e-16,\n -1.9253e-14, -3.7413e-14, 1.4444e-15, 1.2786e-14, -3.6970e-14,\n 9.0944e-16, -8.6545e-15, -5.1383e-15, -1.7143e-15, -2.6186e-14,\n -2.0836e-14, -2.0210e-15, 5.1536e-15, -1.8954e-14, -5.4268e-15,\n -3.3816e-15, -2.9054e-15, 1.0395e-15, 9.0211e-15, -7.0714e-15,\n 4.8219e-16, 9.5817e-16, 2.5376e-16, -2.1262e-14, 1.9713e-15,\n 1.1571e-14, -1.3966e-15, -9.3636e-15, -4.2171e-14, -3.8089e-14,\n -3.6586e-15, 1.8137e-15, 9.9490e-16, 2.6788e-15, -2.3825e-14,\n -3.0363e-15, -1.8008e-15, 8.4445e-15, 1.8324e-15, 6.3065e-16,\n -5.4146e-15, -4.5108e-14, -1.7374e-14, 2.0938e-15, -3.3909e-14,\n -1.4895e-14, -5.1267e-16, 1.0960e-15, 3.6227e-15, -3.9628e-14,\n 6.4546e-16, -1.4925e-14, 1.5222e-14, -2.3540e-14, -3.5550e-15,\n -1.1560e-14, -3.6840e-16, -2.5083e-14, -2.4553e-16, -6.9474e-15,\n 3.6368e-15, 1.2594e-16, 9.4775e-16, -6.4205e-14, -2.4060e-14,\n 1.5674e-15, -3.0558e-14, 3.4463e-14, -9.3253e-15, -1.2261e-15,\n 1.2955e-15, 2.0200e-15, -1.0931e-13, -3.7547e-14, -2.4006e-14,\n -1.9533e-16, -1.1191e-14, -4.9838e-14, -1.9116e-14, -2.2177e-14,\n 1.8455e-15, -2.5799e-14, -1.2434e-14, -2.0475e-14, -1.9877e-15,\n -2.3749e-14, 2.3213e-15, -4.3944e-15, -1.0601e-14, 2.8322e-15,\n -7.8765e-15, 6.3999e-15, 3.0162e-15, -1.4374e-14, -2.6374e-14,\n -5.0413e-14, -2.9368e-14, -1.1253e-14, -4.5980e-15, -3.6968e-15,\n 1.9606e-15, 5.4130e-15, 1.2867e-14, -3.8451e-14, -4.6960e-14,\n -1.0218e-14, 1.2875e-15, -6.8611e-16, -1.5939e-15, -2.4267e-14,\n -1.8683e-14, -8.9881e-14, -1.3725e-14, -1.5952e-14, 2.4660e-15,\n 4.7104e-16, -1.1116e-14, 5.1344e-15, -7.0019e-15, -3.3790e-15,\n -2.3156e-14, -3.5340e-14, -2.2670e-15, 7.0397e-16, 2.6473e-15,\n 1.9921e-15, -2.3243e-14, 1.5967e-14, 6.6735e-15, 1.0351e-15,\n 4.5714e-16, -2.1451e-14, 9.6913e-15, 9.6978e-16, -4.3612e-15,\n 9.9102e-15, -5.2938e-15, 2.5744e-15, -7.6127e-15, 9.3116e-15,\n 5.2315e-15, 8.2385e-17, -1.0329e-16, -4.4324e-15, 5.0603e-15,\n -3.0794e-15, -1.5128e-14, -6.6478e-14, -6.2823e-15, -1.7014e-14,\n -1.5516e-15, 1.4426e-15, 1.9359e-15, 1.4232e-15, 2.3651e-15,\n -8.3630e-15, -6.5989e-16, -1.0093e-14, -2.7231e-15, -3.6573e-14,\n -6.4322e-15, 1.0936e-15, 2.5855e-14, -1.7307e-14, -1.2574e-14,\n -1.9593e-14, 1.3215e-15, -2.6852e-14, -6.4337e-16, -1.5133e-14,\n 2.6247e-16, -1.9967e-14, -1.5170e-15, -2.8934e-14, 5.5812e-15,\n 2.4679e-15, -1.8318e-14, 1.5359e-15, -8.0917e-16, 2.5236e-15,\n 2.4699e-15, 1.0034e-14, -1.8319e-14, -3.3830e-15, -3.6142e-15,\n -2.8509e-14, -5.3249e-14, -4.7366e-15, -1.8339e-14, 3.6996e-17,\n 7.5874e-16, -4.0478e-14, -2.9758e-16, 2.9751e-15, -2.5168e-14,\n -4.0037e-15, 3.2883e-15, -6.5615e-15, -9.5766e-16, -4.8896e-15,\n 1.9232e-14, -9.2022e-15, 7.0423e-15, 1.9023e-14, -3.7919e-15,\n -3.0423e-14, 7.6228e-15, -2.1955e-14, 1.9982e-16, -9.0237e-16,\n -1.2324e-14, -3.1465e-15, -4.7487e-16, -9.5797e-15, -3.4430e-14,\n -4.6406e-15, 4.8757e-15, 6.8743e-15, -4.3344e-14, -1.6593e-14,\n 2.2402e-15, -5.3535e-15, 1.2414e-14, -9.0148e-15, 4.9933e-15,\n 2.2608e-16, -1.6233e-14, -1.0855e-15, -2.8565e-14, -1.2979e-15,\n -3.2364e-16, 6.8986e-16, 6.0046e-15, -1.1282e-14, -4.3416e-15,\n 1.5931e-15, -5.4888e-16, -8.4712e-15, -3.2280e-16, 2.9502e-15,\n -1.9770e-14, -2.9719e-15, -4.0629e-15, -2.2923e-14, -2.2116e-14,\n -2.0259e-14, -5.3528e-16, -3.6255e-15, -5.3539e-15, -9.5495e-16,\n 5.7306e-16, 1.2505e-14, -2.6452e-14, 5.8021e-15, -2.9945e-14,\n 2.0133e-15, -5.4177e-15, 5.3970e-16, -3.7565e-14, -3.4530e-15,\n -5.3969e-14, -1.4703e-14, -3.3172e-15, 6.0482e-15, -2.6620e-14,\n -5.3510e-15, 1.3527e-14, -9.5559e-15, -2.4062e-14, 7.0869e-16,\n -9.9747e-14, 5.6790e-15, -3.3828e-15, 2.6854e-15, 1.4953e-14,\n 3.0600e-15, -5.7887e-16, -1.2134e-14, -5.8313e-15, 1.0060e-15,\n -3.1412e-14, -2.1733e-14, 5.5450e-15, -4.4156e-15, -3.9811e-15,\n 5.5629e-15, -3.2732e-14, 6.3484e-15, -9.1945e-15, -2.8168e-14,\n 9.6588e-16, -2.3264e-14, -6.5149e-15, -6.2372e-15, -1.6215e-14,\n -4.0633e-15, 1.1460e-17, -5.0792e-14, 3.4292e-14, -7.8453e-15,\n 1.4017e-15, 3.0459e-15, -1.4713e-14, -3.9072e-14, 9.0351e-15,\n 5.9012e-15, -3.4106e-15, -1.5728e-14, 3.5985e-15, 3.0157e-15,\n -3.5024e-14, 6.9945e-16, -1.6578e-14, -1.4924e-14, 1.9781e-15,\n -2.1305e-14, 3.9534e-15, -2.6634e-14, -7.8182e-15, -8.3690e-15,\n 2.5671e-14, -2.9125e-15, -2.0338e-15, -6.9701e-14, -1.4757e-14,\n 3.4802e-15, -1.9353e-15, -1.4459e-14, 4.0773e-15, 1.2986e-15,\n -5.9018e-14, 1.7210e-14, -5.0249e-15, -8.1337e-14, -3.8531e-15,\n -3.9136e-15, 1.5964e-15, 3.8063e-15, -6.4612e-15, -2.7654e-15,\n -4.1126e-14, -1.4615e-14, 3.0879e-14, 6.0551e-16, -9.3769e-15,\n -7.6926e-15, -5.7036e-14, -3.3617e-15, -1.2130e-14, -3.6493e-15,\n 5.5180e-16, -4.7033e-15, -2.0537e-15, 2.0416e-16, -7.4660e-14,\n 1.1562e-14, 7.8978e-15, -2.3920e-14, 1.2741e-14, -4.4759e-14,\n -2.3738e-14, 5.4487e-15, -4.3314e-17, -1.5237e-14, -3.9269e-15,\n -2.4263e-14, 1.0658e-15, 4.9414e-15, -9.1144e-15, 9.9691e-16,\n -9.8798e-15, -5.5287e-14, 2.7903e-15, -1.3633e-14, -4.0776e-14,\n -5.3841e-15, 6.0180e-16, 7.7492e-15, -1.2048e-15, -1.5552e-15,\n -1.0993e-14, 1.4507e-15, -4.2915e-14, -3.9755e-15, -2.2314e-14,\n 3.1321e-15, -5.5520e-15, -3.2662e-14, 9.9160e-16, -2.9043e-14,\n -2.0586e-15, 1.6455e-14, 6.9243e-15, -1.7955e-14, -1.3300e-14,\n -2.1368e-14, -1.7078e-14, -8.4954e-14, 1.3949e-15, 2.5601e-15,\n 5.5083e-15, -8.2289e-15, 2.2941e-14, -4.1409e-14, 8.8339e-16,\n -1.0317e-14, -2.0979e-15, -4.8346e-14, -4.2371e-14, -2.4605e-14,\n 3.3013e-15, -3.0221e-14, -1.2277e-15, -9.7968e-16, -4.1273e-15,\n -1.1425e-16, -3.4407e-16, -2.5863e-14, -1.3543e-14, -6.3605e-15,\n -4.8101e-14, -9.1197e-17, -6.5150e-14, 1.7183e-15, -1.2203e-14,\n 3.4336e-15, -3.7290e-16, 6.9228e-15, -5.2966e-15, -2.9887e-15,\n -3.2196e-15, -1.2301e-14, 2.1437e-16, 2.1447e-15, -4.2788e-14,\n -6.8987e-15, -6.2687e-14, 5.5586e-15, -6.6406e-14, 1.1835e-14,\n 1.9526e-16, -2.9581e-14, -6.2010e-17, -1.2886e-14, 3.0603e-15,\n 1.8645e-16, -8.1915e-15, -5.1093e-14, -4.4268e-14, -6.2109e-14,\n -3.4299e-15, -2.1815e-14, -3.4897e-14, -3.9379e-15, 1.0200e-15,\n -3.1139e-15, 2.2308e-14, -1.4031e-14, -1.4235e-14, -1.6666e-15,\n -3.1461e-14, -9.7799e-15, 1.6658e-15, -3.9227e-16, -2.3135e-14,\n -1.0080e-15, -2.2562e-14, 1.1139e-14, -1.5733e-14, 4.9804e-15,\n -2.5009e-14, 7.7638e-15, -2.5768e-14, -2.6091e-14, -1.2491e-15,\n 6.9962e-15, -4.3118e-14, -1.8661e-14, 4.6662e-16, -4.8475e-14,\n -2.8589e-14, -3.3199e-15, 1.7306e-15, -1.6891e-14, 6.0753e-16,\n -7.2810e-16, -1.8799e-14, -8.3135e-15, -1.5468e-15, -2.4101e-15,\n -1.7913e-14, -9.1452e-16, -2.2401e-14, 7.5099e-16, -3.2341e-14,\n 9.3092e-16, -1.2742e-15, -3.7280e-15, -5.8564e-15, -1.1234e-15,\n -2.1630e-15, -1.2911e-14, -3.3854e-14, 7.2731e-15, 1.5056e-16,\n -7.2969e-15, -1.0893e-15, 6.4462e-16, 6.9259e-15, -5.6265e-14,\n -7.1129e-16, -2.3889e-16, -2.5928e-14, -6.2298e-14, -3.6880e-14,\n -4.6982e-14, 6.8113e-15, -4.5779e-15, 1.9829e-14, 1.2414e-14,\n 1.7567e-15, -1.6412e-14, -7.3327e-16, -1.0126e-14, -6.8569e-14,\n -7.0769e-15, -3.4810e-14, -4.9551e-15, -1.0659e-15, -6.5954e-14,\n -8.1292e-14, -1.4876e-14, -1.5929e-15, -5.1579e-15, 1.4981e-14,\n -1.6734e-14, 1.3209e-15, 1.8402e-16, -1.4595e-14, -2.5528e-15,\n 8.6199e-16, -1.9397e-15, -4.4139e-16, 4.2325e-16, 7.4363e-16,\n 7.0533e-15, -1.7079e-14, 2.1660e-16, 4.4415e-15, -5.2822e-17,\n -2.7052e-15, -2.4594e-14, -4.8196e-14, -4.4010e-15, -1.6150e-14,\n 1.0179e-15, -5.8746e-15, -1.1323e-15, -2.1958e-14, 4.5465e-15,\n -4.5567e-14, -6.0166e-15, 2.0553e-15, -2.0795e-14, -8.3854e-14,\n 5.6222e-15, 1.3646e-15, -2.5517e-14, -2.2562e-14, 1.7956e-15,\n 2.1689e-15, -4.0093e-14, -4.2822e-14, 9.7991e-15, 1.2179e-14,\n -2.5370e-15, -8.0849e-15, -1.0464e-14, 1.1187e-15, 1.6188e-15,\n -9.9491e-15, -1.0461e-15, -3.1504e-14, -5.3938e-15, 8.5150e-15,\n -5.2775e-14, 1.2674e-16, -1.9168e-14, 1.1581e-15, -5.4202e-15,\n -2.1383e-14, -9.6137e-15, 5.3048e-15, -7.2134e-15, -5.0501e-14,\n 2.6728e-15, -1.8372e-16, 9.2919e-16, 1.6779e-15, 9.5802e-15,\n -3.6654e-14, -2.5555e-14, -4.2807e-15, -3.8375e-14, -1.4821e-15,\n -9.3280e-16, 5.7605e-15, -2.9302e-14, -1.9918e-15, -6.5273e-15,\n -1.6892e-14, 2.3816e-15, -1.3544e-14, -1.1485e-15, -2.2147e-14,\n -6.7295e-16, 6.5437e-15, 6.8489e-16, -2.6290e-14, -7.5490e-15,\n 6.8897e-16, -1.2023e-14, -3.9711e-14, -2.6421e-14, -6.2722e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([6.8068e-11, 5.3715e-12, 4.0100e-13, 7.1580e-12, 3.5978e-12, 4.7736e-12,\n 1.6149e-10, 2.2305e-10, 8.1977e-11, 1.4693e-13, 2.7242e-10, 1.8492e-11,\n 4.5732e-10, 4.5708e-11, 3.7261e-12, 1.7211e-10, 3.8749e-11, 1.1758e-10,\n 1.6697e-11, 6.2261e-11, 4.7311e-11, 1.5412e-12, 8.3917e-12, 1.9411e-13,\n 3.9504e-10, 3.9516e-11, 1.8871e-12, 3.1950e-10, 2.0360e-11, 4.9412e-11,\n 6.3925e-12, 1.2070e-12, 5.1076e-13, 5.2095e-12, 1.8708e-12, 2.2002e-11,\n 6.1071e-12, 2.5877e-13, 6.5231e-12, 1.5757e-12, 1.8680e-11, 2.0005e-12,\n 5.4787e-11, 1.9068e-11, 1.5152e-11, 2.2645e-11, 3.9519e-13, 9.4714e-14,\n 1.3122e-10, 2.6122e-11, 1.6559e-12, 1.7106e-11, 2.8163e-11, 3.0856e-11,\n 3.3623e-12, 9.5685e-12, 7.4383e-11, 1.4092e-10, 3.0765e-10, 9.3861e-11,\n 2.3398e-12, 8.7374e-13, 1.1101e-10, 2.2886e-12, 9.7175e-11, 1.2500e-11,\n 8.0359e-12, 3.6646e-10, 4.4881e-10, 2.0217e-10, 2.4819e-11, 1.2742e-12,\n 8.5734e-11, 3.0385e-13, 3.1159e-12, 1.3190e-12, 7.7337e-13, 3.0661e-11,\n 5.1671e-11, 2.1415e-10, 6.4527e-11, 4.4694e-12, 8.6933e-11, 5.7595e-11,\n 2.0602e-11, 7.2743e-12, 3.3783e-12, 7.1930e-12, 8.8821e-11, 1.8005e-10,\n 1.8777e-12, 5.7935e-12, 1.9978e-12, 2.5597e-11, 3.9477e-11, 1.1131e-12,\n 2.8060e-12, 4.1807e-10, 5.8372e-11, 5.9132e-10, 3.4137e-12, 7.7897e-11,\n 6.2005e-12, 2.5718e-11, 1.3283e-12, 1.1512e-11, 2.0773e-11, 9.4723e-12,\n 7.9378e-13, 3.7735e-12, 2.5625e-10, 1.1230e-10, 4.1298e-10, 2.1220e-10,\n 3.2583e-11, 6.7356e-13, 2.6579e-12, 1.9229e-10, 9.5358e-12, 5.9837e-11,\n 1.6898e-12, 4.5595e-11, 1.8725e-13, 5.3398e-13, 1.2255e-10, 1.0179e-11,\n 1.5745e-10, 2.2798e-11, 5.1369e-11, 2.1665e-11, 1.2477e-13, 5.4920e-11,\n 4.4379e-11, 1.6869e-10, 1.4303e-10, 4.8097e-10, 2.4561e-12, 1.0540e-12,\n 1.2188e-11, 2.8880e-11, 6.2755e-13, 9.6436e-12, 1.6672e-11, 3.1178e-11,\n 5.6986e-12, 1.0581e-12, 4.2401e-12, 2.7090e-10, 3.4971e-12, 2.0648e-12,\n 1.5836e-11, 2.1814e-11, 8.9632e-11, 2.7716e-12, 7.4931e-11, 7.7392e-13,\n 4.3139e-12, 1.9029e-10, 2.9187e-11, 1.3409e-11, 1.8426e-10, 9.1147e-11,\n 3.2302e-10, 1.3739e-11, 1.2354e-11, 3.2667e-10, 1.3389e-12, 4.2880e-11,\n 8.5485e-11, 2.0387e-10, 6.3665e-12, 2.3880e-10, 1.2775e-11, 4.2897e-13,\n 2.3526e-10, 1.9615e-11, 7.7240e-13, 2.5808e-10, 2.6679e-11, 1.5735e-11,\n 2.0068e-10, 2.0626e-11, 3.8465e-11, 1.9903e-13, 2.1606e-11, 8.8762e-14,\n 2.7363e-10, 3.2046e-11, 9.4733e-12, 3.5103e-13, 2.9526e-11, 9.6629e-13,\n 3.8000e-12, 2.4819e-13, 3.3023e-10, 1.5345e-12, 3.5631e-11, 1.3419e-11,\n 6.5317e-10, 1.6236e-10, 4.5152e-12, 8.3441e-11, 1.1871e-10, 2.2711e-10,\n 1.2503e-12, 4.9035e-13, 2.6418e-10, 3.2692e-13, 2.1785e-11, 1.6749e-10,\n 2.1836e-10, 2.2745e-12, 1.8759e-12, 3.1689e-11, 6.2138e-10, 1.0823e-10,\n 3.0210e-11, 1.1229e-12, 2.3394e-11, 2.9520e-11, 3.3014e-11, 7.2683e-11,\n 3.4112e-10, 3.1959e-12, 8.2683e-13, 4.5696e-12, 8.6189e-10, 3.0180e-12,\n 6.8329e-11, 1.0232e-11, 1.6849e-12, 1.2193e-12, 2.6854e-10, 2.2091e-11,\n 1.4948e-10, 2.1718e-10, 2.2611e-13, 2.9725e-10, 3.0718e-12, 2.8586e-11,\n 3.3790e-13, 1.1419e-10, 2.0482e-10, 2.6937e-11, 9.7960e-13, 4.6254e-12,\n 5.9656e-13, 5.0548e-11, 1.5118e-11, 5.0073e-11, 2.6017e-12, 6.0708e-13,\n 8.5677e-13, 1.3504e-12, 1.9743e-13, 3.4144e-12, 3.2062e-12, 8.2076e-13,\n 1.6838e-11, 6.8327e-11, 1.8096e-10, 6.5757e-13, 2.0334e-11, 5.5055e-13,\n 2.5242e-12, 7.1127e-11, 1.0898e-11, 7.4720e-12, 2.9817e-11, 2.1994e-10,\n 1.4482e-11, 3.8923e-13, 2.7223e-12, 1.8390e-12, 5.3740e-11, 2.6006e-11,\n 3.5633e-12, 8.7800e-14, 1.1111e-10, 8.0247e-11, 3.4189e-12, 1.1884e-10,\n 9.7513e-13, 7.5854e-10, 1.5628e-11, 2.0157e-12, 1.2295e-13, 3.2703e-13,\n 2.3104e-10, 1.8264e-11, 4.4631e-12, 3.3634e-12, 3.4394e-11, 9.7509e-12,\n 7.9306e-13, 2.6677e-12, 1.7709e-10, 5.0827e-11, 2.4616e-11, 1.7623e-11,\n 2.0143e-10, 1.0396e-11, 1.3694e-10, 1.4863e-10, 1.0469e-10, 9.4557e-13,\n 1.6215e-12, 1.5083e-11, 1.2619e-11, 4.7771e-11, 4.5923e-11, 2.7316e-12,\n 9.5597e-11, 4.5475e-11, 2.2069e-10, 1.5112e-13, 2.6779e-13, 5.7855e-11,\n 2.8117e-10, 1.7221e-10, 3.1946e-13, 2.1986e-11, 2.7701e-11, 1.2841e-12,\n 1.4639e-12, 5.1347e-11, 3.8152e-11, 3.0660e-12, 5.4582e-11, 1.5288e-12,\n 1.7656e-10, 8.0273e-12, 1.2550e-10, 3.7228e-11, 2.3684e-10, 8.6580e-12,\n 1.7969e-10, 2.0852e-13, 2.4491e-10, 4.7909e-10, 1.3776e-10, 2.1991e-11,\n 2.2336e-12, 4.0912e-12, 7.6315e-12, 1.7468e-10, 9.3253e-12, 3.1439e-10,\n 2.8029e-11, 5.2023e-11, 6.2794e-11, 1.8104e-13, 6.0534e-13, 2.5039e-13,\n 2.9122e-12, 3.4049e-11, 3.0303e-10, 4.2960e-11, 6.7340e-12, 6.8083e-11,\n 6.4339e-11, 1.9696e-10, 1.1965e-11, 8.8357e-11, 2.3104e-10, 2.5052e-11,\n 4.4581e-11, 5.4613e-11, 6.6557e-12, 8.0275e-11, 5.4164e-10, 3.2780e-12,\n 2.4109e-10, 3.7084e-10, 2.9805e-12, 5.1320e-11, 4.4166e-12, 1.4602e-12,\n 7.9724e-13, 2.9374e-10, 1.0685e-10, 1.2716e-10, 1.3455e-10, 2.8788e-12,\n 7.3362e-14, 1.2286e-10, 1.8308e-12, 6.4918e-13, 2.4148e-10, 1.2670e-10,\n 1.6999e-10, 2.0844e-13, 2.4038e-10, 3.0790e-12, 2.9182e-13, 2.7046e-13,\n 4.7062e-13, 8.6237e-11, 1.1001e-11, 4.2172e-10, 1.4457e-13, 9.2070e-13,\n 6.6261e-10, 1.2580e-10, 3.0960e-11, 5.5497e-12, 4.5697e-13, 1.0165e-13,\n 1.7946e-10, 5.9459e-11, 3.7455e-11, 1.0204e-13, 1.9681e-11, 6.8881e-12,\n 3.8279e-12, 3.7769e-12, 6.2410e-13, 2.1962e-10, 5.8912e-10, 1.0299e-12,\n 1.9951e-12, 6.8564e-13, 2.2881e-10, 6.8693e-11, 3.4492e-10, 7.6376e-11,\n 2.3770e-12, 1.2059e-12, 4.2807e-12, 1.1828e-11, 1.8081e-13, 7.1088e-11,\n 1.3569e-11, 1.6141e-10, 1.1961e-11, 3.2061e-10, 5.8407e-13, 1.1719e-10,\n 2.4024e-13, 6.6708e-13, 3.0774e-12, 3.3570e-12, 2.1691e-10, 3.5762e-12,\n 1.4733e-11, 4.4967e-11, 1.8390e-10, 3.4940e-11, 1.4849e-12, 1.6505e-12,\n 3.6003e-13, 7.5999e-12, 3.1548e-13, 1.2376e-10, 3.3569e-11, 5.0423e-13,\n 1.5063e-10, 1.4592e-12, 2.0573e-10, 2.8053e-13, 2.7958e-11, 3.0916e-11,\n 1.1969e-10, 4.2570e-12, 3.0304e-10, 5.1217e-12, 3.2255e-10, 1.2666e-12,\n 1.8825e-11, 1.6366e-13, 3.3180e-11, 5.6345e-11, 1.6365e-11, 5.8080e-12,\n 2.9487e-11, 1.0372e-10, 1.6885e-12, 1.2903e-12, 1.6124e-12, 4.7538e-11,\n 1.2170e-11, 1.2596e-12, 4.9295e-10, 1.4819e-10, 1.3826e-11, 5.6111e-10,\n 1.2369e-10, 1.2387e-10, 6.9635e-12, 7.2861e-13, 4.7217e-11, 2.5004e-12,\n 1.4777e-12, 2.4126e-11, 4.9277e-10, 2.2238e-11, 1.0689e-12, 7.2293e-12,\n 4.2051e-10, 2.5263e-13, 6.5187e-12, 3.4579e-10, 2.7257e-11, 2.1867e-10,\n 9.7864e-11, 1.7565e-11, 1.1095e-12, 1.0600e-11, 6.2085e-12, 6.8518e-11,\n 1.3756e-13, 6.0258e-13, 2.2065e-11, 1.2377e-10, 9.4671e-12, 7.2050e-12,\n 4.1404e-12, 1.2776e-11, 5.6125e-10, 6.7142e-13, 2.9659e-10, 1.5903e-13,\n 4.2250e-13, 8.8470e-11, 9.6938e-11, 1.1311e-11, 7.7807e-13, 4.7907e-10,\n 1.2578e-10, 1.9242e-10, 2.1441e-10, 2.8949e-10, 4.5519e-11, 5.5666e-11,\n 1.2246e-10, 2.6530e-10, 9.6965e-11, 7.4765e-12, 6.4417e-11, 4.5487e-11,\n 1.5256e-12, 9.2538e-11, 1.4025e-10, 2.5124e-12, 2.3009e-10, 2.9731e-13,\n 1.2055e-11, 5.3110e-12, 1.6783e-11, 6.9485e-12, 6.9217e-12, 1.5327e-10,\n 3.7152e-10, 7.1575e-11, 1.1828e-10, 1.2015e-10, 1.3798e-11, 1.0803e-12,\n 2.9163e-11, 1.6421e-12, 3.2233e-12, 1.9427e-12, 3.5858e-13, 2.0347e-10,\n 4.0245e-12, 5.1639e-12, 4.7986e-11, 5.0071e-11, 4.0356e-11, 3.5209e-10,\n 4.5574e-12, 1.2867e-10, 5.5588e-12, 1.2905e-10, 6.2619e-12, 5.0574e-11,\n 6.6207e-12, 2.4984e-12, 2.6727e-11, 2.0653e-09, 1.6422e-13, 5.5545e-13,\n 1.1891e-12, 1.0755e-12, 1.6294e-12, 2.5300e-12, 4.9778e-12, 1.4246e-11,\n 2.7007e-11, 1.8412e-12, 7.8417e-13, 2.4107e-12, 5.7077e-13, 3.2857e-12,\n 5.9275e-12, 6.5523e-13, 1.3218e-11, 9.4817e-12, 5.8265e-13, 6.2965e-12,\n 7.6516e-11, 1.8236e-11, 1.7285e-11, 1.5323e-13, 6.2332e-11, 2.3231e-12,\n 8.7686e-12, 6.8700e-11, 1.1054e-10, 2.3993e-10, 1.8975e-11, 5.2636e-13,\n 8.4830e-12, 8.9605e-13, 1.1374e-09, 1.4035e-11, 1.6702e-12, 3.7909e-12,\n 4.5558e-12, 4.4460e-10, 6.3592e-14, 1.0070e-11, 8.6963e-12, 1.0732e-11,\n 1.2778e-10, 2.4812e-10, 4.8868e-11, 3.7217e-11, 1.0141e-11, 1.1668e-11,\n 1.5508e-10, 1.3377e-11, 7.6003e-13, 7.8064e-11, 1.6762e-10, 1.8222e-12,\n 7.3670e-11, 9.7869e-12, 1.2895e-12, 4.6187e-12], device='cuda:0')" + }, + "54": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.2222e-17, 6.8702e-17, -5.6052e-45, ..., 2.2758e-15,\n -9.6299e-15, -5.6052e-45],\n [-4.7070e-17, -3.4579e-16, -5.6052e-45, ..., -1.3688e-15,\n -3.1578e-14, -5.6052e-45],\n [-2.9054e-17, 7.5700e-19, 5.6052e-45, ..., -2.2195e-15,\n -7.3728e-15, -5.6052e-45],\n ...,\n [-1.0743e-16, -5.7284e-17, 5.6052e-45, ..., -2.2801e-15,\n -3.2698e-14, -5.6052e-45],\n [-1.8730e-17, 7.6548e-17, 5.6052e-45, ..., 9.1077e-15,\n -1.5873e-15, 5.6052e-45],\n [-3.4408e-16, -1.1850e-16, 5.6052e-45, ..., -9.8888e-15,\n -7.5216e-14, -5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.6355e-12, 2.2612e-12, 2.5382e-16, ..., 2.8579e-12, 5.4717e-13,\n 1.5935e-14],\n [2.0786e-11, 1.9847e-11, 1.1372e-16, ..., 2.6842e-11, 5.6950e-12,\n 9.5510e-14],\n [1.4786e-12, 1.6486e-12, 1.0998e-14, ..., 1.7198e-12, 3.6283e-13,\n 1.1105e-13],\n ...,\n [1.2270e-11, 2.2872e-11, 1.5000e-16, ..., 2.3211e-11, 3.9070e-12,\n 4.1404e-14],\n [6.0852e-12, 4.7222e-12, 1.9752e-16, ..., 1.1842e-11, 3.3430e-12,\n 3.2286e-14],\n [1.9058e-10, 1.6404e-10, 6.6804e-15, ..., 2.4231e-10, 5.6698e-11,\n 1.4829e-13]], device='cuda:0')" + }, + "55": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-8.4421e-13, -3.4004e-12, -6.6064e-13, -2.1630e-12, 1.0535e-12,\n 1.2646e-12, -1.9772e-12, -5.0314e-12, -1.1344e-12, 8.3202e-13,\n 2.1338e-12, 2.8421e-13, 6.6385e-13, 2.8301e-12, -8.4322e-13,\n 9.3490e-13, 3.0661e-12, -1.1439e-12, -7.9864e-13, -4.5234e-12,\n 9.1342e-13, -8.3924e-12, 1.1314e-12, -2.3962e-12, 3.5222e-13,\n 2.3481e-13, -5.6528e-13, 1.0595e-12, 2.7206e-12, -1.0647e-12,\n 1.0360e-12, 1.0756e-12, -1.0287e-12, -7.8816e-13, 1.7641e-12,\n 3.3574e-13, -9.5075e-13, -8.3263e-13, 3.1033e-13, 2.2953e-12,\n 7.1783e-13, 5.7714e-13, -5.3399e-13, 2.4934e-12, -5.5199e-12,\n 7.5920e-13, -1.5236e-12, 2.1611e-12, 1.0949e-12, -2.8230e-12,\n 1.8974e-13, -1.5860e-12, 2.3350e-12, 5.8701e-13, 1.2450e-12,\n 6.6176e-14, 1.2214e-12, 4.6650e-13, 1.7892e-14, -9.3165e-13,\n -2.9762e-12, 1.8479e-12, -3.2179e-13, 4.7118e-12, 1.3556e-13,\n 6.5914e-13, 7.9129e-13, 1.6742e-13, -7.3358e-13, 8.1132e-13,\n -1.3077e-12, -3.4153e-14, -1.2635e-12, -1.4395e-12, -4.9924e-12,\n -9.1411e-13, -4.1390e-13, -1.2441e-14, -4.5555e-12, 6.1531e-13,\n 1.1138e-12, -1.4623e-12, 3.4213e-13, -6.9410e-13, -5.7862e-13,\n -1.9539e-13, 1.9684e-12, -1.2755e-12, 2.4628e-12, -9.7276e-13,\n -9.6601e-13, -2.7072e-12, -1.5862e-12, 8.9042e-13, 2.6974e-12,\n 4.8358e-13, 1.2764e-13, 2.2539e-13, 5.8133e-13, -1.8891e-12,\n 8.9520e-13, 6.6400e-13, -7.5857e-14, -1.1326e-12, 3.9015e-13,\n -2.6951e-12, 1.3802e-13, -4.6892e-13, 2.4710e-12, -1.3480e-12,\n -4.6026e-12, 1.2347e-13, 3.8212e-12, 1.0684e-12, 3.0714e-13,\n -1.9124e-12, 5.3799e-13, 1.0059e-12, 9.9417e-13, -1.7925e-12,\n 5.3048e-13, 2.6356e-13, 5.7740e-13, 2.5932e-14, -3.6943e-12,\n 1.2109e-12, 5.7291e-13, 4.0654e-12, -2.4549e-13, 1.1389e-12,\n 9.5529e-13, 4.1287e-12, 7.2791e-13, 2.2790e-14, 2.8352e-12,\n -1.2675e-13, 1.6037e-12, 9.5804e-13, -9.2357e-13, 2.0161e-13,\n 9.1011e-13, -1.5363e-14, 6.2382e-13, -3.2394e-13, 1.3675e-12,\n 2.3500e-13, -1.7098e-12, 1.3309e-12, 1.1403e-12, 9.7824e-13,\n -1.5094e-13, -7.6999e-13, 1.6391e-12, -4.2200e-13, 1.4878e-13,\n -1.3471e-12, 1.4537e-12, 3.7678e-13, -2.0599e-14, -7.6629e-13,\n 6.8910e-13, -1.1626e-12, -3.9303e-12, -3.9342e-12, -2.3964e-13,\n 7.8354e-13, -2.1193e-12, -8.5007e-13, -2.0130e-13, 2.0488e-13,\n 7.1104e-13, 3.8896e-13, 3.8052e-13, -1.5873e-12, 1.0413e-12,\n 1.3681e-12, -1.8565e-12, 1.6683e-12, -4.1801e-13, 6.3726e-13,\n -1.2864e-12, -3.0328e-15, -3.1366e-12, -6.4440e-13, -9.3966e-13,\n 1.6219e-12, -2.7395e-12, 8.6092e-13, 6.5690e-13, 6.9970e-13,\n -1.1050e-13, 9.6251e-13, 2.8198e-12, 2.3339e-12, 7.2404e-13,\n -6.3662e-13, 3.7764e-12, 1.0920e-12, 2.9065e-13, -1.9406e-13,\n -2.3449e-12, 2.0516e-12, 5.3690e-13, -3.3900e-13, 8.2114e-13,\n 7.8545e-13, -8.5553e-15, 1.0346e-12, -6.4411e-13, 1.9372e-12,\n -3.2214e-13, -9.4155e-13, 2.5578e-13, 3.2139e-12, -4.7980e-13,\n 1.0329e-12, -1.8951e-13, 1.9364e-12, 1.1514e-12, 9.9283e-13,\n 3.1101e-12, 1.1340e-12, -1.7138e-12, 1.3103e-12, -2.6463e-13,\n 2.1268e-12, 1.5720e-12, -6.3949e-12, 1.4543e-12, 8.5689e-13,\n 9.8233e-14, 4.1977e-12, -8.6072e-13, 1.8106e-12, 1.6050e-12,\n 1.0415e-12, 1.8192e-12, 6.4512e-13, -3.1838e-12, -2.7513e-13,\n 1.6916e-12, -1.4551e-12, 5.2995e-13, 1.2040e-13, -1.6030e-12,\n 1.3329e-12, 2.2925e-12, 2.1956e-12, 6.2090e-14, 6.1332e-13,\n 2.5724e-13, 7.9979e-13, -1.6975e-13, -3.5417e-12, 3.4389e-13,\n 1.2938e-12, -1.7548e-12, 1.3199e-12, 2.1294e-12, 6.3478e-14,\n -2.7941e-12, 1.6280e-12, 1.0071e-12, 3.0225e-13, 1.9516e-12,\n -6.9193e-12, 7.1310e-13, 5.3979e-13, 9.5049e-13, -7.0848e-12,\n -1.0678e-12, 2.1623e-12, 1.4514e-13, -7.5872e-13, 1.6221e-12,\n 1.3758e-12, 9.6608e-15, -3.9682e-13, 4.2080e-13, -5.9120e-13,\n -4.5928e-12, 1.0186e-14, 1.2044e-12, -7.1575e-12, 8.6384e-13,\n -3.5005e-12, -1.4040e-12, 1.0858e-12, -5.5224e-12, -2.5532e-12,\n -2.8356e-12, -3.7622e-12, 6.0820e-13, 2.4233e-12, 4.4325e-12,\n -9.9729e-13, -2.2024e-12, 6.4083e-13, 5.5792e-13, 1.1734e-13,\n 1.0246e-12, 1.1112e-12, -3.7929e-13, 2.0077e-13, -3.5928e-12,\n 1.2268e-15, -3.6615e-13, 2.4651e-12, -2.0329e-12, 7.6670e-14,\n 1.5755e-12, 2.6224e-12, -4.7748e-12, 9.1935e-13, 1.9630e-12,\n 1.3814e-12, 7.0999e-13, -2.7177e-12, 7.8375e-13, -4.7462e-13,\n 2.5069e-12, 9.9619e-13, 2.6554e-12, -1.6289e-13, 1.0214e-12,\n -7.6296e-13, 1.6661e-12, -4.6720e-13, 3.1937e-12, 3.1327e-13,\n 1.1624e-12, -2.3938e-12, -3.1547e-13, -1.9930e-13, 1.6359e-13,\n 1.2853e-12, 2.8400e-12, 1.6488e-13, -4.7884e-12, -5.7610e-13,\n 1.4426e-13, -4.7353e-13, 1.9799e-12, 1.5079e-12, -2.3092e-13,\n 8.4932e-13, 9.9624e-13, 1.3124e-12, 5.9827e-12, -4.6543e-13,\n 5.3988e-13, 1.5918e-12, -2.5963e-13, 1.0807e-12, 2.6889e-12,\n 1.1736e-12, -9.3476e-13, 4.0666e-12, -1.7444e-13, -5.5342e-12,\n -2.0088e-12, 3.8379e-13, -8.6716e-13, -2.3390e-12, 2.7834e-12,\n 3.3729e-12, 1.2485e-13, -1.8068e-12, 8.0931e-13, -6.4194e-12,\n 2.6773e-12, -7.3345e-13, -7.7667e-13, -1.5091e-13, 2.5778e-13,\n 3.2771e-13, 2.2727e-12, 3.7205e-13, -2.4173e-12, 1.5476e-12,\n 3.6612e-13, -2.4155e-12, 3.3986e-13, 1.7127e-12, 1.9440e-12,\n 1.8999e-12, 2.3612e-13, 1.6031e-12, -2.7988e-12, -1.3428e-13,\n 1.1368e-12, 1.1397e-12, 1.4363e-12, 5.2866e-13, -4.4879e-14,\n 2.7786e-13, -6.6152e-13, 4.2800e-13, -9.0494e-12, -1.8393e-12,\n 1.5729e-12, 4.4589e-12, -1.3927e-11, 2.5614e-12, -3.7831e-12,\n -3.9843e-14, 1.1310e-13, -1.3539e-12, -3.4392e-12, -9.2437e-13,\n -3.2557e-12, 2.7082e-12, -4.8124e-13, 6.4682e-12, 2.9865e-12,\n 1.3997e-12, -5.1213e-12, -4.1337e-13, -5.6283e-12, 7.2120e-12,\n 3.5046e-12, 1.2127e-12, 1.4798e-12, -1.0377e-13, 1.0683e-12,\n 3.6919e-13, -2.1846e-12, 9.7557e-13, -4.0322e-12, 2.4879e-12,\n -3.6425e-12, 5.8385e-13, 4.7670e-13, 1.6029e-12, 2.2612e-12,\n -5.8500e-13, 1.5393e-13, -1.3500e-12, -9.7905e-13, 1.7412e-12,\n -3.0037e-13, 6.4991e-13, -1.0700e-12, -5.7337e-12, -1.8146e-12,\n 1.1393e-12, 4.3092e-12, -2.3339e-12, -1.1877e-12, 5.1507e-13,\n 1.1683e-12, 2.7557e-12, -6.8178e-13, -4.2751e-12, -8.8670e-13,\n -6.8890e-13, -1.4604e-12, 3.7649e-14, 1.4412e-13, -1.0489e-12,\n 9.6872e-13, 7.0013e-14, -1.1630e-12, -1.1378e-12, 3.6812e-13,\n 1.6216e-12, 3.2210e-13, -1.6890e-12, 7.6402e-13, 1.3243e-12,\n 1.0528e-12, 1.2939e-12, -1.8764e-13, -4.7323e-13, -8.2580e-14,\n -3.2373e-12, 2.5227e-12, 2.7786e-12, 3.1487e-12, -4.1861e-12,\n 1.1170e-12, 2.7284e-13, 1.9391e-12, 7.3481e-14, 3.6450e-13,\n 1.9849e-13, 1.9674e-12, 1.5304e-12, 6.2006e-13, -6.7767e-13,\n -2.0990e-12, 2.0431e-12, 7.7767e-13, 6.6476e-13, 1.0088e-12,\n 9.7233e-13, -3.7045e-12, 6.2521e-13, 4.9851e-13, 3.8267e-12,\n 2.3511e-12, 2.3342e-13, -1.3457e-13, 7.5107e-13, -4.8345e-13,\n -3.5054e-13, 3.9856e-13, -4.9991e-12, 5.8379e-13, -3.8864e-12,\n 1.8286e-12, 2.2305e-13, 2.6733e-12, -5.9643e-15, 1.3129e-13,\n -8.4346e-13, 3.2848e-13, -8.1914e-13, 2.5271e-12, -5.2158e-13,\n 1.5792e-12, 4.2357e-15, 1.9848e-13, -8.6467e-13, 3.8214e-13,\n 5.5762e-13, 7.7441e-13, 1.2547e-12, 3.5653e-13, 1.5549e-12,\n 1.9211e-13, 6.9626e-13, 1.2185e-12, 7.2675e-13, 8.1782e-13,\n 2.4807e-13, 1.4341e-12, -2.2790e-12, 8.1568e-13, -2.5972e-12,\n -4.8520e-12, -1.5325e-12, 1.9076e-12, 9.1227e-13, -2.5503e-13,\n 1.6961e-12, -2.3162e-13, -1.1560e-13, -3.3071e-13, 3.6549e-13,\n -5.7317e-12, 1.1546e-13, 1.8042e-12, 1.8328e-12, 1.7157e-12,\n -4.9791e-13, 9.0115e-13, -1.6457e-12, 1.4705e-12, 1.0157e-12,\n 1.4325e-13, -3.8238e-12, 1.8820e-12, -1.2127e-14, -1.9066e-12,\n -2.7584e-13, 6.9634e-13, -1.0020e-11, 2.0181e-13, -9.1463e-13,\n 1.8976e-13, 5.9713e-13, -1.1006e-12, 5.5257e-13, 4.5144e-13,\n 3.0486e-12, 5.4174e-13, 6.7404e-13, -1.1290e-12, -6.2655e-13,\n -1.1583e-13, 8.2022e-13, -3.3776e-12, -1.2503e-12, -5.4341e-12,\n 1.1935e-12, -8.9129e-13, 1.1426e-12, -1.7164e-13, 3.3402e-12,\n 1.3123e-12, 1.8332e-12, -3.8097e-12, -2.1742e-13, -1.1770e-12,\n -2.9019e-12, 6.1347e-14, 2.4732e-13, -5.3052e-13, 3.4379e-13,\n -1.6812e-12, 1.0384e-12, 1.6229e-12, -7.4563e-16, -2.7413e-13,\n -3.8733e-13, -8.2700e-12, 6.5478e-13, 6.9068e-13, 2.2972e-13,\n 6.1921e-13, 2.3412e-12, 5.7402e-12, -1.0327e-12, 3.3940e-12,\n 2.8823e-12, -1.4954e-12, 3.0204e-13, -1.1622e-12, -4.2064e-13,\n 6.4866e-13, 2.0944e-12, -4.0277e-12, 5.0924e-13, 6.1978e-12,\n 2.9746e-12, -4.0069e-13, -7.9368e-13, 3.8357e-13, -5.7127e-12,\n 5.6101e-13, 7.9089e-13, 1.9253e-13, 6.3081e-13, 1.2536e-12,\n 1.7651e-12, 1.7917e-12, -2.6212e-12, 3.1524e-14, -7.3416e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.3951e-08, 7.4778e-07, 8.1242e-08, 1.1231e-06, 8.0820e-07, 1.6087e-08,\n 1.8490e-08, 2.9779e-06, 1.1250e-07, 1.2286e-07, 1.2527e-06, 1.3890e-06,\n 1.8767e-06, 2.1899e-09, 9.1664e-08, 6.7665e-08, 2.7032e-08, 9.6238e-08,\n 6.4381e-07, 1.0828e-07, 2.0817e-07, 9.3531e-07, 4.1962e-09, 6.2702e-08,\n 7.5606e-07, 8.6524e-08, 1.6082e-06, 7.1269e-08, 1.0988e-08, 9.9255e-09,\n 4.3371e-08, 1.4049e-08, 2.8989e-08, 2.9022e-08, 6.9582e-09, 2.1799e-08,\n 1.4458e-06, 4.7390e-09, 9.6715e-09, 2.3506e-09, 2.1404e-07, 4.2493e-09,\n 9.8760e-08, 6.0386e-08, 4.4698e-07, 1.9521e-09, 2.5307e-07, 3.0807e-09,\n 3.5254e-09, 1.8094e-06, 3.3535e-08, 9.1498e-07, 1.8610e-09, 2.1131e-06,\n 3.2898e-09, 5.5011e-08, 1.1057e-06, 2.3644e-09, 1.3578e-08, 4.4794e-07,\n 9.3231e-07, 2.3887e-06, 4.4574e-08, 6.0387e-09, 9.5372e-07, 2.0782e-08,\n 2.3682e-07, 1.8871e-06, 1.6261e-07, 5.9874e-06, 2.9099e-07, 1.4622e-06,\n 3.4591e-07, 5.7372e-07, 5.5963e-07, 7.6338e-09, 1.4532e-07, 2.3760e-08,\n 4.9310e-06, 1.6734e-07, 1.0295e-07, 2.5137e-07, 1.1303e-07, 6.5064e-07,\n 1.0651e-08, 3.9449e-09, 1.9630e-06, 1.1317e-08, 5.7307e-09, 2.1091e-07,\n 6.6082e-09, 1.2078e-07, 1.5935e-08, 2.2978e-06, 2.7746e-08, 8.2340e-08,\n 4.6688e-09, 7.1386e-09, 3.8448e-08, 7.7672e-07, 5.7621e-07, 3.8645e-07,\n 1.6182e-06, 6.2352e-09, 6.9869e-07, 1.1999e-06, 1.9366e-07, 1.2950e-06,\n 4.9376e-09, 1.9549e-07, 3.3515e-06, 6.3564e-07, 8.9997e-06, 1.6308e-06,\n 3.4400e-08, 2.5175e-06, 3.8994e-07, 2.8345e-06, 1.2430e-08, 2.4167e-07,\n 2.7544e-07, 9.3928e-08, 2.6141e-09, 2.0950e-08, 1.5218e-06, 2.8714e-07,\n 5.7765e-07, 4.9577e-07, 8.1969e-08, 1.9210e-08, 7.1405e-09, 1.0057e-06,\n 2.7452e-06, 1.0708e-07, 1.8346e-06, 2.2247e-08, 1.9372e-06, 2.3910e-07,\n 2.4725e-08, 1.4566e-07, 6.3595e-08, 3.4901e-07, 6.9039e-07, 5.1482e-08,\n 5.0077e-07, 5.6155e-07, 5.7536e-07, 5.8261e-08, 1.5075e-07, 5.3608e-08,\n 4.0079e-09, 5.7277e-09, 1.7556e-06, 1.3173e-06, 1.0111e-07, 3.1871e-07,\n 7.1723e-07, 5.4857e-08, 1.1722e-07, 1.3726e-07, 1.0181e-07, 7.8887e-09,\n 7.8215e-06, 3.1364e-06, 1.2563e-08, 2.3772e-07, 1.3771e-07, 9.5655e-09,\n 2.9625e-07, 1.2102e-06, 1.2879e-07, 2.1913e-07, 7.8566e-07, 1.0904e-07,\n 6.4945e-07, 5.1385e-07, 5.1609e-08, 1.0253e-06, 2.8339e-07, 6.3488e-07,\n 1.9252e-06, 2.7830e-07, 4.9818e-07, 2.0182e-07, 8.1606e-08, 7.2436e-07,\n 5.6184e-06, 5.0536e-07, 8.2571e-09, 7.4487e-07, 8.8385e-09, 5.8526e-09,\n 2.7302e-08, 2.8456e-08, 4.9054e-07, 9.3508e-07, 3.6700e-08, 4.4021e-07,\n 3.0235e-06, 3.7792e-08, 1.2797e-07, 5.0591e-07, 3.5756e-06, 2.5136e-08,\n 4.8039e-09, 1.4161e-07, 2.4784e-07, 2.2986e-08, 4.2528e-06, 1.5273e-07,\n 2.7347e-08, 1.1703e-07, 1.8290e-08, 1.0820e-08, 4.2486e-09, 1.8286e-07,\n 1.2557e-08, 2.9170e-09, 6.0006e-09, 3.0860e-08, 2.9918e-09, 2.1104e-08,\n 1.4840e-07, 5.8878e-08, 4.8016e-09, 1.1796e-09, 6.4352e-08, 3.6606e-06,\n 4.6829e-08, 2.4495e-07, 8.4484e-07, 8.3248e-08, 1.4359e-06, 2.2511e-07,\n 2.5425e-06, 6.5840e-07, 3.0062e-08, 7.1751e-07, 1.3548e-06, 5.6209e-09,\n 3.8703e-09, 5.4300e-07, 2.3350e-06, 8.5890e-09, 5.6892e-07, 1.8732e-07,\n 4.3303e-07, 2.1425e-08, 7.8027e-07, 1.7411e-07, 2.0724e-06, 1.5995e-06,\n 3.6286e-08, 2.5706e-06, 4.2155e-07, 1.4933e-07, 3.4461e-07, 4.4295e-07,\n 9.4010e-09, 2.7087e-08, 5.9839e-07, 2.0936e-08, 7.1960e-07, 4.7598e-09,\n 2.4007e-08, 7.2912e-07, 7.1615e-07, 2.6894e-08, 3.6530e-08, 4.4418e-06,\n 3.4002e-09, 3.0268e-09, 6.2685e-08, 9.2120e-09, 1.6336e-07, 1.9925e-09,\n 2.2501e-07, 3.3933e-07, 1.4837e-06, 2.5693e-08, 1.2664e-08, 4.7766e-08,\n 2.2852e-06, 3.1153e-06, 1.0060e-07, 7.4630e-07, 1.8141e-07, 7.9903e-08,\n 1.9679e-06, 1.2744e-06, 8.8916e-07, 7.9917e-07, 9.5998e-07, 4.2602e-09,\n 7.3056e-08, 2.4731e-07, 1.4727e-08, 8.9027e-07, 5.8851e-07, 1.7849e-08,\n 2.4409e-06, 1.4019e-07, 4.0346e-08, 2.5034e-06, 7.4091e-07, 2.3133e-08,\n 9.9234e-09, 9.3551e-07, 6.5429e-07, 2.5980e-08, 1.1204e-06, 7.0498e-09,\n 4.6544e-06, 2.3334e-07, 2.6480e-06, 3.5818e-07, 1.1104e-08, 3.2505e-07,\n 1.1142e-06, 3.0796e-08, 3.7195e-08, 2.3549e-07, 6.8614e-09, 2.2646e-07,\n 1.7659e-06, 4.3769e-09, 3.7037e-09, 4.2208e-07, 6.4692e-09, 5.3832e-07,\n 2.6412e-09, 4.6483e-07, 1.2585e-07, 1.9323e-08, 3.0886e-08, 3.6821e-08,\n 1.2444e-06, 9.1927e-09, 7.0960e-07, 4.1552e-07, 6.7756e-07, 7.3993e-09,\n 5.6369e-08, 5.0817e-09, 9.0837e-08, 2.8850e-07, 2.9193e-07, 6.2025e-07,\n 9.9741e-08, 2.2100e-07, 8.0355e-09, 2.3978e-09, 5.6439e-09, 4.5560e-07,\n 4.8960e-09, 1.4008e-06, 2.4585e-08, 2.1713e-08, 3.6795e-07, 5.6591e-06,\n 6.2913e-08, 7.0192e-09, 2.0168e-08, 6.4089e-09, 1.3625e-07, 6.2499e-09,\n 1.5197e-07, 6.8655e-07, 1.9599e-06, 1.8008e-06, 5.6680e-07, 1.9258e-07,\n 5.3392e-09, 1.1664e-07, 1.0527e-07, 1.6628e-07, 1.2764e-07, 3.9215e-07,\n 8.9090e-07, 5.2254e-08, 1.9421e-07, 3.7514e-08, 2.8432e-09, 6.1944e-08,\n 6.8975e-09, 3.1842e-06, 9.3811e-08, 6.5884e-08, 3.5428e-06, 3.8004e-07,\n 6.5713e-07, 1.0965e-08, 7.4344e-06, 1.1849e-06, 1.5217e-08, 1.2947e-06,\n 7.5725e-08, 4.9684e-07, 4.5939e-06, 8.5879e-08, 2.3448e-07, 7.3140e-07,\n 1.5494e-05, 1.3460e-06, 2.1446e-07, 3.5011e-09, 1.8842e-07, 2.4209e-07,\n 7.9581e-07, 2.2173e-08, 1.7997e-07, 3.5328e-09, 5.8578e-07, 1.7982e-07,\n 1.0511e-08, 3.2856e-09, 3.2990e-06, 7.7875e-08, 1.3856e-06, 2.1458e-07,\n 2.3876e-09, 4.1711e-08, 6.5481e-09, 8.4478e-07, 5.4280e-07, 1.3758e-07,\n 6.6391e-07, 1.8880e-09, 6.6715e-06, 7.0772e-06, 2.3578e-07, 6.9608e-07,\n 4.2792e-07, 4.6279e-09, 6.8646e-08, 6.4946e-07, 1.5908e-07, 1.2459e-08,\n 2.8394e-09, 2.8095e-09, 8.0648e-08, 2.0578e-06, 4.6240e-08, 4.2032e-07,\n 4.2791e-08, 1.5563e-06, 7.5738e-06, 4.4791e-07, 4.0096e-07, 6.9348e-09,\n 5.6685e-09, 8.4851e-09, 1.3939e-08, 9.0453e-07, 1.2416e-07, 6.8057e-09,\n 1.0507e-07, 7.1762e-07, 2.0996e-08, 5.6174e-08, 1.2860e-06, 1.1018e-06,\n 5.5129e-07, 1.2643e-08, 7.6778e-08, 3.0837e-06, 6.0546e-08, 2.1510e-07,\n 8.0429e-09, 1.2665e-07, 2.3429e-08, 1.3822e-06, 3.3361e-08, 3.5375e-07,\n 3.6976e-08, 2.3858e-06, 3.8212e-08, 2.8779e-08, 3.2080e-09, 2.1163e-06,\n 4.8421e-09, 1.9686e-07, 7.9611e-06, 4.7545e-07, 6.6162e-09, 6.1117e-08,\n 2.5650e-08, 3.8489e-09, 6.9380e-09, 7.6794e-07, 6.7641e-08, 5.4193e-07,\n 1.1478e-07, 1.0244e-08, 3.1532e-06, 5.5617e-07, 3.9681e-07, 5.3079e-09,\n 3.9236e-08, 7.0526e-09, 3.1490e-08, 1.6027e-07, 1.5019e-08, 3.8413e-08,\n 1.7861e-06, 6.9803e-08, 8.6585e-09, 1.3397e-06, 6.0659e-07, 4.2944e-07,\n 3.8320e-07, 5.7250e-08, 2.0217e-07, 4.0253e-08, 2.1328e-06, 4.2795e-08,\n 1.5658e-07, 3.4357e-07, 1.5233e-07, 3.7963e-08, 2.4950e-06, 7.4186e-09,\n 3.3787e-09, 3.0412e-08, 1.1413e-06, 1.3754e-06, 8.8782e-09, 8.2614e-08,\n 5.5500e-08, 2.4015e-08, 3.8581e-07, 1.7432e-07, 6.3412e-08, 4.6294e-07,\n 1.7540e-07, 1.4296e-08, 2.8105e-07, 1.2231e-06, 4.9492e-09, 2.6460e-07,\n 1.8133e-07, 5.8494e-07, 5.1101e-08, 9.6320e-08, 1.4924e-08, 1.6975e-09,\n 2.3750e-07, 7.3810e-07, 4.7635e-08, 3.7002e-07, 5.8368e-06, 4.2344e-08,\n 3.8038e-09, 3.4350e-09, 2.0571e-06, 9.6968e-07, 8.9978e-09, 7.3382e-07,\n 5.0465e-07, 1.3483e-07, 5.2046e-09, 2.8148e-06, 1.0117e-08, 3.0631e-07,\n 5.3398e-08, 1.4656e-08, 1.3373e-08, 4.2601e-06, 4.3397e-07, 3.0714e-07,\n 8.6932e-07, 1.8715e-07, 1.9714e-08, 2.0890e-06, 4.3881e-07, 2.6793e-08,\n 4.0253e-07, 1.4509e-07, 1.0196e-07, 1.1250e-07, 2.9140e-07, 1.2341e-07,\n 8.9867e-07, 5.4841e-07, 1.2049e-06, 3.3382e-07, 7.1118e-09, 9.0533e-07,\n 1.4081e-07, 2.8096e-06, 1.1439e-08, 1.1998e-07, 5.7222e-07, 1.3845e-07,\n 2.1169e-08, 1.6222e-06, 1.1080e-06, 7.3838e-08, 2.9758e-07, 1.7736e-09,\n 3.8705e-06, 1.5658e-08, 3.8051e-09, 2.8919e-07, 3.6022e-06, 4.9726e-08,\n 2.6259e-06, 4.9608e-09, 2.9344e-09, 4.0425e-08, 1.2092e-08, 7.0779e-08,\n 5.3579e-09, 2.6604e-08, 2.5077e-08, 6.8614e-09, 1.8882e-06, 2.0770e-07,\n 1.4009e-06, 2.3195e-06, 3.7039e-09, 1.2605e-07, 4.4303e-07, 5.1868e-07,\n 1.2609e-07, 7.3032e-06, 4.2177e-08, 2.2994e-06, 8.5556e-08, 2.5939e-06,\n 3.1006e-07, 2.2255e-06, 3.2878e-07, 2.6670e-09, 3.6363e-06, 2.0950e-07,\n 7.6743e-07, 7.5210e-07, 5.0443e-07, 7.6283e-06], device='cuda:0')" + }, + "56": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 6.5536e-15, -1.5438e-13, -6.2252e-15, -6.1755e-14, -3.1069e-13,\n 5.6280e-16, -1.4067e-14, -2.2738e-13, -4.2832e-15, 2.4597e-15,\n -1.5382e-16, 4.6453e-15, -7.1230e-16, 3.5663e-15, -8.1773e-14,\n -4.7661e-14, 4.5755e-15, 1.4651e-14, 5.5933e-15, -5.5060e-13,\n -8.8316e-14, -2.5761e-13, -2.4876e-16, -2.8822e-14, -3.8950e-16,\n -1.0502e-15, 2.2989e-15, -8.0649e-15, 2.9524e-15, 4.6282e-15,\n -2.4150e-15, -1.3740e-15, 9.8701e-15, -3.6155e-14, -7.5647e-15,\n -2.4681e-15, -6.9108e-15, -1.0391e-15, -8.6612e-16, -1.6003e-14,\n -6.7849e-16, -6.0395e-16, -1.9530e-15, -1.5145e-13, -5.5591e-13,\n 3.1665e-15, -5.1889e-14, -4.0461e-16, 1.1087e-15, -1.6475e-13,\n 1.1978e-15, 5.7334e-15, -5.3749e-15, 1.9324e-15, 1.6933e-15,\n -4.0465e-14, 7.5577e-15, -3.0032e-15, -1.2787e-14, -1.4719e-14,\n -2.7804e-13, -1.0560e-15, -5.9871e-16, 2.5168e-14, -5.3277e-15,\n -3.4683e-16, 6.8086e-15, 6.1695e-15, 1.2570e-15, -2.1646e-15,\n -2.0575e-14, 3.7355e-17, 1.2078e-14, -1.6624e-13, -1.7275e-13,\n 7.3257e-15, 1.3341e-14, 1.1817e-15, -2.2843e-13, -7.1020e-14,\n 1.6489e-15, -6.2012e-13, -3.4834e-16, -6.9280e-14, 7.3489e-15,\n -4.5404e-16, -4.4572e-15, -2.1199e-14, -1.1622e-15, -1.0650e-13,\n 7.9416e-16, -2.8522e-13, -8.0139e-15, -2.5881e-14, -2.5845e-15,\n -1.3897e-14, -1.3694e-15, 2.9465e-15, -5.0226e-15, 1.4185e-14,\n -1.0995e-13, -1.2382e-14, -8.2972e-15, -1.1839e-14, -6.2871e-14,\n -7.1783e-14, -1.0356e-15, 3.1907e-15, 7.8422e-15, -5.5951e-15,\n -3.1512e-13, -2.5525e-14, 7.5030e-15, 6.2065e-16, 2.3845e-15,\n -1.9034e-13, -6.5492e-14, -9.1129e-16, -1.1208e-15, -9.9327e-15,\n -6.6295e-15, -4.3372e-14, -7.1191e-16, 4.0379e-16, -2.9030e-13,\n -2.2166e-16, -2.2662e-13, -1.4173e-13, -6.4626e-16, -5.3688e-16,\n -5.3171e-16, 2.5944e-14, -4.1051e-15, -2.4996e-15, -3.6060e-15,\n -8.0805e-15, -7.9856e-15, -2.3186e-14, -7.7419e-15, 1.2837e-16,\n -8.6061e-16, -8.4835e-15, 5.4415e-15, 1.6701e-15, 1.8142e-15,\n -7.4967e-14, -3.7990e-14, -9.2696e-15, -1.2868e-15, 2.1728e-15,\n -7.6469e-16, -1.2218e-14, 6.4182e-15, 8.6601e-16, -9.8707e-16,\n 1.2450e-14, -1.4864e-13, -1.4302e-15, 2.5076e-15, -1.9841e-16,\n 9.8767e-16, 1.0354e-14, -3.6897e-13, -2.5251e-13, 1.2240e-16,\n -1.7570e-15, -5.7041e-13, 1.0361e-14, -1.2097e-14, -3.5699e-15,\n -1.3618e-13, 1.5000e-15, -5.3020e-14, -1.0015e-14, -1.1222e-13,\n -6.8596e-15, 4.8134e-15, -5.1841e-16, -1.3071e-14, -8.9166e-16,\n -1.0607e-13, -1.4379e-15, -1.1324e-13, 2.9994e-16, -2.6642e-14,\n -2.0911e-14, -5.5556e-13, 3.6313e-15, -3.8627e-16, 1.2457e-15,\n 6.2888e-15, -4.3603e-16, -4.1460e-15, 2.9289e-15, -1.7263e-16,\n -1.1863e-13, 1.4252e-14, -4.4929e-14, 6.5542e-15, 3.1361e-15,\n -2.3700e-13, -1.1114e-13, 5.1196e-16, -3.6116e-14, 2.2050e-16,\n 3.1019e-16, -1.9523e-13, -2.0216e-13, 1.1883e-15, 2.0329e-15,\n 4.2302e-16, 2.4566e-15, 3.4764e-15, 1.1231e-15, 4.4676e-15,\n 2.4977e-17, -6.2332e-15, 1.1574e-15, -4.2589e-15, 6.4838e-16,\n 4.8573e-15, -5.3141e-15, -2.7810e-13, -1.6359e-15, 2.9815e-16,\n -1.6328e-15, 2.3369e-15, -1.6162e-13, 1.3013e-15, -1.6253e-14,\n 7.7863e-18, 2.1787e-15, -3.1653e-14, -1.7647e-13, -4.2999e-16,\n -1.2309e-15, -6.2297e-15, 6.9458e-16, -2.6589e-13, -1.9652e-15,\n -8.4418e-15, -2.3261e-13, 3.4839e-15, 5.5496e-17, 3.8493e-15,\n 6.8417e-16, -2.3346e-14, -3.2967e-17, -4.7117e-15, 1.6425e-15,\n -6.9171e-15, 6.8553e-16, -6.0736e-15, -9.4388e-14, 1.2785e-15,\n 1.9977e-17, -1.7105e-13, -8.7331e-14, -4.3799e-14, -9.6884e-16,\n -1.1587e-13, -3.9830e-14, -1.6555e-13, 2.6603e-15, 2.9315e-15,\n -3.1084e-13, 3.2255e-15, -1.5266e-15, -6.9614e-15, -4.3613e-13,\n 4.3411e-15, -7.2029e-15, -8.5127e-16, -1.2567e-14, 2.3314e-15,\n 2.3259e-15, -1.2243e-13, -7.3775e-14, 9.9447e-16, 1.0911e-14,\n -3.0122e-15, -2.6773e-15, -1.9136e-14, -4.3942e-13, -1.0686e-14,\n -1.9377e-13, -7.0365e-14, 3.2831e-15, -2.6262e-13, -5.5068e-13,\n -3.3382e-13, -2.7413e-13, 3.3485e-15, 1.5454e-15, 2.0530e-14,\n 3.8619e-15, -7.3756e-15, -5.0192e-15, -4.6933e-16, -9.3372e-16,\n 1.8303e-16, -2.9447e-15, -2.6004e-15, -9.9851e-15, -4.2857e-13,\n -2.2715e-16, 6.8398e-16, -9.8241e-15, -6.7319e-14, -2.3726e-16,\n -2.2850e-14, 2.9649e-15, -3.7178e-13, -3.6587e-16, 3.8690e-15,\n -9.8339e-16, -9.6119e-17, -8.0836e-14, -2.8672e-14, 8.1253e-15,\n -2.7884e-13, -1.7288e-15, -2.8505e-15, -3.5374e-16, -3.4141e-14,\n -1.4096e-15, -1.7374e-16, -2.7866e-15, 1.5144e-15, -7.6036e-16,\n 7.9102e-17, -1.6327e-13, -2.5828e-13, 1.1294e-15, 6.3729e-17,\n -1.1226e-15, 1.2470e-14, -5.5659e-15, -1.4485e-13, 8.3451e-15,\n -4.8137e-14, -3.4541e-15, -3.5241e-15, 5.4340e-17, 1.3335e-16,\n -1.7301e-13, -1.0246e-16, -1.2453e-15, -3.8146e-13, -4.8151e-14,\n 2.5499e-15, -1.0512e-15, -1.0224e-14, -8.7578e-15, -6.4171e-15,\n -7.7099e-14, -1.8866e-14, -1.4326e-14, 8.9480e-17, -3.3046e-13,\n -9.5181e-14, 9.6781e-16, 1.9630e-15, -4.8292e-15, 2.9985e-15,\n 3.3031e-14, -1.3848e-15, 2.2492e-15, -1.3355e-15, -2.4945e-13,\n 1.7570e-14, 1.0194e-15, -8.9320e-15, -1.3438e-15, -2.2876e-13,\n -5.6986e-13, -9.5491e-15, 4.8875e-15, -1.2054e-13, -6.1180e-15,\n 4.1638e-15, 4.1541e-15, -9.1730e-16, -9.6298e-14, -1.4299e-15,\n -1.3535e-14, 2.3668e-15, 1.0103e-15, -3.3423e-13, -1.9442e-14,\n 7.4804e-16, -2.2731e-15, 7.0316e-16, -1.8392e-16, 2.4060e-15,\n -1.6020e-15, 9.8826e-16, -1.3002e-13, -3.4092e-13, -2.5173e-14,\n -1.9616e-15, 3.9288e-14, -4.7165e-13, -1.1327e-14, -2.3494e-13,\n 5.7293e-17, -2.4903e-15, 8.2075e-15, -4.6669e-13, 4.0659e-15,\n -6.7253e-13, -2.0688e-15, 6.6078e-15, -1.2467e-14, 2.2246e-14,\n 1.9891e-16, -2.1715e-13, 3.0273e-15, -1.3995e-13, -1.7354e-14,\n 1.4167e-14, -6.0074e-15, 1.0377e-15, -1.8726e-13, 6.3656e-16,\n 3.4239e-15, -2.4454e-13, 8.1458e-15, -2.6310e-13, 2.5930e-15,\n -1.7667e-13, -7.9085e-16, -3.1969e-14, -1.9816e-15, -2.5605e-13,\n -1.0513e-14, 1.4919e-16, 6.4282e-15, 5.0296e-15, 2.7817e-16,\n 4.9191e-16, 3.3991e-15, 1.3073e-14, -2.8235e-13, -7.7129e-15,\n -2.2208e-15, 1.2790e-14, -1.1746e-14, -1.3133e-13, 6.6072e-15,\n -1.0426e-13, -2.9601e-15, -4.6050e-15, -2.8712e-13, 1.9495e-16,\n -1.4895e-14, -4.0550e-13, -5.6868e-15, 1.4391e-15, -1.3560e-14,\n -1.1879e-15, 2.2787e-15, -1.1107e-13, -1.0334e-14, -1.7031e-14,\n -5.0898e-15, 6.1681e-16, -3.1703e-13, -2.3421e-15, 2.4536e-15,\n 6.2552e-17, -1.2669e-15, -1.4347e-13, -4.5527e-13, 1.4106e-14,\n -1.4423e-13, -9.7904e-14, -6.0588e-15, 5.3617e-15, -1.2552e-13,\n -5.5599e-15, 8.0570e-15, 1.0922e-15, 2.2513e-15, 9.3183e-17,\n -1.7264e-15, -1.8519e-15, -2.2261e-15, -4.0814e-15, -1.2371e-13,\n 2.3846e-14, -1.2216e-13, -3.5646e-13, -4.4630e-15, 2.7409e-15,\n -2.1094e-14, -5.0033e-13, -8.6747e-16, -2.6409e-15, -7.9860e-14,\n 4.0389e-15, 1.1350e-15, 4.7066e-16, -1.8977e-16, -2.5247e-14,\n -2.0708e-15, 5.7828e-15, -2.3809e-13, -6.7957e-14, -1.9238e-13,\n -6.6352e-15, -8.6525e-16, -1.3975e-13, 5.9124e-15, 1.3208e-16,\n 7.8994e-15, -1.1401e-15, 1.3012e-14, -8.6154e-15, -2.2560e-16,\n -9.1717e-16, 3.7741e-16, 5.0785e-16, -4.9254e-15, 1.5968e-15,\n -2.7729e-13, -1.1164e-15, -1.4802e-14, 3.6018e-15, 9.6731e-16,\n -1.6152e-13, -3.9669e-15, -2.9714e-15, -1.3057e-15, -8.1344e-15,\n 1.0522e-15, -1.3951e-15, 1.3171e-14, -2.1424e-15, -1.8620e-14,\n -2.3798e-13, -1.9528e-13, -6.6050e-14, 9.1608e-16, -7.8895e-15,\n 2.1215e-15, 5.5304e-15, 5.1024e-15, -7.3325e-14, -4.9034e-16,\n -2.6833e-13, 4.5693e-16, -6.7618e-17, -5.4620e-15, -2.9509e-16,\n 8.8339e-15, -3.5125e-15, -2.3369e-13, -1.9923e-13, -1.9573e-15,\n 2.3585e-15, -9.3012e-14, -5.4501e-16, 1.6478e-15, 1.7527e-14,\n 2.8679e-15, -3.1017e-15, -3.2212e-13, 5.3732e-15, 6.5027e-15,\n 2.8837e-15, -3.8527e-15, 1.3957e-15, 9.2678e-16, -3.3737e-13,\n -1.0173e-14, -2.3091e-16, -8.7376e-15, -3.6858e-13, -3.4355e-14,\n -6.0170e-15, -2.2859e-13, -2.3991e-13, -6.9134e-14, -7.5765e-13,\n -1.5557e-15, -9.7501e-15, -6.2977e-14, 2.4288e-17, -9.8134e-15,\n 1.0789e-14, -4.9811e-13, -3.6932e-13, 8.1528e-16, 2.2707e-15,\n -2.6466e-13, -1.5637e-17, 8.9334e-15, -2.1359e-13, -1.1725e-15,\n -2.2289e-13, 5.8378e-15, -9.6797e-15, -1.0327e-14, -3.0581e-16,\n -1.5780e-14, -5.6294e-13, -1.9248e-15, -3.4733e-15, 4.7669e-15,\n 2.5872e-15, 4.4529e-16, 3.3560e-14, -1.0747e-14, 4.2200e-14,\n 1.3776e-14, -1.4089e-13, -6.6222e-15, -3.6885e-13, -5.2163e-14,\n 3.1714e-15, -2.6736e-13, -1.8437e-13, 1.0506e-16, 9.1527e-15,\n 7.2243e-15, -3.0237e-15, -9.2027e-14, -1.9580e-15, -2.1372e-13,\n -9.5524e-16, 3.7580e-15, -3.2140e-13, 3.3653e-16, 1.3354e-15,\n 3.5947e-16, -2.4637e-14, -7.1496e-14, -9.1576e-13, -3.1304e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.8102e-11, 6.1040e-11, 1.5919e-12, 6.4302e-10, 3.0814e-10, 3.1924e-13,\n 1.0722e-11, 3.8380e-10, 5.6804e-12, 4.8619e-12, 7.5051e-11, 2.2076e-10,\n 3.1302e-10, 7.1901e-13, 3.5182e-11, 7.7670e-13, 2.5338e-13, 3.5524e-11,\n 3.0307e-12, 2.4288e-11, 1.9886e-10, 3.2534e-10, 1.0456e-12, 1.1828e-11,\n 1.4795e-10, 1.2716e-12, 5.0685e-11, 1.1763e-12, 2.7304e-13, 1.1475e-13,\n 1.6372e-11, 2.0050e-13, 9.1924e-12, 1.6549e-11, 1.1795e-13, 2.1548e-12,\n 6.9554e-10, 1.1851e-12, 3.8511e-13, 4.3776e-13, 5.7057e-12, 8.0577e-14,\n 3.3327e-11, 9.4287e-12, 1.9576e-10, 5.1459e-13, 1.3987e-11, 7.9521e-13,\n 6.6062e-14, 2.2496e-10, 8.3055e-13, 9.8797e-12, 3.8862e-14, 7.8003e-10,\n 3.6255e-13, 1.2303e-11, 8.4071e-11, 7.9703e-14, 2.2038e-13, 1.8365e-11,\n 6.0357e-11, 5.8640e-10, 5.6502e-13, 9.1803e-13, 2.9673e-10, 3.8911e-13,\n 1.9659e-12, 3.5827e-10, 9.4905e-12, 1.4841e-09, 9.6702e-13, 1.3544e-10,\n 3.3025e-12, 1.8539e-11, 3.6835e-10, 3.0658e-13, 1.8172e-11, 2.4550e-13,\n 1.6243e-09, 1.7370e-10, 3.1431e-11, 1.4570e-10, 5.1774e-12, 1.8748e-11,\n 3.7003e-13, 2.4377e-12, 2.8258e-10, 5.8001e-12, 1.2295e-13, 1.5179e-10,\n 2.5294e-13, 3.4130e-11, 1.2459e-11, 2.7913e-10, 1.0852e-11, 3.1178e-12,\n 1.5660e-12, 2.9995e-12, 4.2535e-12, 3.6553e-12, 4.7584e-11, 2.5307e-11,\n 5.9844e-10, 2.0191e-12, 4.3749e-11, 9.4610e-10, 2.1954e-11, 5.0756e-11,\n 9.5331e-13, 7.4188e-11, 1.3938e-10, 3.1213e-11, 2.4480e-09, 1.1854e-10,\n 5.2894e-13, 2.3810e-10, 1.4680e-10, 2.1107e-10, 4.1758e-12, 7.1610e-11,\n 3.2578e-12, 4.8772e-13, 8.2718e-14, 1.4158e-12, 1.3890e-10, 2.8509e-11,\n 2.0515e-10, 3.3605e-10, 2.6772e-12, 1.0049e-13, 2.2867e-12, 2.4204e-11,\n 5.6479e-11, 9.3119e-12, 2.8229e-11, 1.1065e-11, 6.8240e-11, 4.0362e-11,\n 1.3704e-12, 1.3493e-12, 7.6362e-12, 2.6079e-11, 4.1991e-11, 8.2651e-12,\n 6.1329e-11, 7.2356e-10, 2.8352e-10, 1.9652e-11, 1.6511e-12, 3.9066e-13,\n 5.2048e-13, 7.4769e-13, 2.4508e-11, 2.2609e-10, 3.7379e-12, 3.0399e-12,\n 4.5214e-10, 2.0356e-11, 1.2269e-11, 3.2639e-11, 4.4334e-13, 2.5539e-12,\n 2.0851e-09, 2.5797e-09, 8.5998e-13, 1.5846e-10, 8.6018e-11, 2.3350e-12,\n 1.7303e-11, 9.1347e-11, 1.4920e-10, 5.8303e-12, 1.4598e-11, 6.2930e-13,\n 5.8028e-11, 6.2613e-12, 1.7365e-12, 5.3344e-11, 8.1479e-11, 1.2050e-10,\n 3.4057e-10, 4.8305e-12, 9.5899e-12, 5.5583e-11, 4.8038e-12, 8.4891e-11,\n 5.1430e-09, 1.2853e-11, 2.9343e-12, 3.2137e-10, 1.4542e-12, 1.3170e-13,\n 1.1727e-11, 8.3282e-13, 4.8782e-12, 1.9857e-10, 1.2658e-12, 9.8895e-11,\n 1.0926e-09, 1.3694e-11, 1.0188e-11, 3.7879e-11, 4.4608e-10, 1.2804e-11,\n 1.6167e-13, 1.8917e-12, 4.0963e-12, 1.0848e-12, 6.2495e-10, 1.0948e-12,\n 1.4221e-11, 1.3854e-11, 3.3573e-13, 3.9895e-13, 2.8337e-13, 3.4210e-12,\n 3.3602e-12, 5.8753e-13, 1.2065e-12, 6.7085e-12, 3.6962e-13, 5.0831e-13,\n 3.7549e-12, 9.5796e-13, 1.9283e-12, 3.8840e-13, 6.5054e-13, 1.2785e-09,\n 2.1825e-13, 2.4907e-11, 1.9436e-10, 4.3037e-12, 1.0958e-09, 1.2205e-11,\n 2.4741e-10, 4.1375e-12, 4.4651e-13, 1.6441e-10, 1.7302e-10, 9.6505e-14,\n 1.0041e-12, 2.0433e-10, 1.2476e-10, 3.9612e-14, 2.0398e-11, 2.6686e-12,\n 2.1860e-10, 8.0192e-13, 9.2434e-11, 1.0496e-12, 7.3485e-10, 4.0835e-10,\n 1.1800e-12, 2.0372e-10, 2.3568e-10, 1.4578e-12, 8.1495e-12, 1.7925e-10,\n 7.3162e-13, 4.5952e-13, 1.1981e-10, 3.5594e-13, 6.6620e-10, 1.6371e-12,\n 1.5394e-12, 5.6557e-11, 1.2955e-10, 6.2504e-13, 9.2111e-12, 7.9971e-10,\n 2.0047e-12, 8.8137e-14, 2.9555e-12, 5.8980e-12, 2.9799e-12, 4.1405e-14,\n 5.7479e-11, 1.6704e-10, 9.7327e-11, 9.1569e-13, 8.0102e-12, 5.9796e-13,\n 1.6439e-10, 3.4262e-10, 9.0177e-13, 2.6937e-11, 8.2749e-11, 6.0768e-12,\n 4.0986e-10, 2.7295e-10, 2.6406e-11, 7.3427e-11, 1.9675e-10, 5.6952e-13,\n 8.0014e-13, 3.7508e-12, 9.9204e-12, 2.0887e-10, 2.9301e-10, 1.5809e-12,\n 2.0488e-09, 5.4455e-12, 2.1746e-11, 9.4747e-10, 1.6067e-10, 6.8368e-13,\n 4.2113e-12, 7.9234e-12, 4.1430e-12, 1.9176e-13, 1.0437e-10, 1.2004e-12,\n 9.9623e-10, 6.8782e-12, 6.0038e-10, 6.5795e-11, 3.1574e-12, 3.5624e-12,\n 2.8832e-10, 1.6818e-12, 7.2247e-11, 1.6074e-11, 6.4652e-13, 2.1957e-12,\n 3.0440e-10, 6.4434e-13, 9.0780e-13, 6.2880e-12, 3.8207e-14, 8.1852e-12,\n 7.7652e-13, 1.6046e-10, 4.7139e-12, 5.2329e-13, 5.6985e-13, 7.0984e-13,\n 3.4372e-11, 9.4981e-14, 1.1847e-11, 1.3054e-11, 1.2391e-11, 1.0382e-13,\n 3.1315e-12, 2.7429e-12, 4.8452e-13, 8.7371e-12, 6.8685e-12, 4.0724e-12,\n 1.1342e-10, 1.1539e-12, 5.8766e-13, 1.3174e-12, 1.4420e-12, 1.6975e-10,\n 7.1056e-14, 9.5105e-10, 2.4125e-12, 6.9511e-13, 1.4955e-11, 2.7508e-09,\n 2.6945e-12, 1.7851e-12, 1.1768e-12, 3.4082e-12, 1.1880e-12, 7.8464e-14,\n 4.3439e-11, 1.0645e-10, 3.4361e-11, 4.6524e-11, 9.3142e-12, 5.8202e-12,\n 1.7960e-12, 9.6598e-13, 4.3434e-11, 9.1129e-11, 1.1975e-12, 4.4512e-11,\n 1.3732e-10, 8.2587e-13, 6.7982e-11, 9.8715e-13, 4.9481e-13, 1.5242e-11,\n 1.6042e-13, 1.1923e-09, 1.5622e-11, 6.0182e-12, 1.7537e-09, 2.8734e-11,\n 1.6577e-10, 3.2896e-13, 2.1209e-09, 9.1931e-11, 2.7581e-13, 2.4573e-11,\n 1.2603e-12, 3.6523e-10, 1.4946e-10, 1.5475e-12, 4.4643e-12, 8.4729e-12,\n 3.4100e-09, 2.9865e-11, 1.5810e-11, 1.7716e-12, 1.4659e-11, 2.6981e-10,\n 1.0914e-10, 3.6792e-13, 2.0957e-10, 4.7943e-13, 1.9561e-10, 9.1226e-11,\n 2.5814e-12, 4.0653e-13, 3.2080e-10, 6.0531e-13, 1.1383e-10, 6.9471e-11,\n 8.6652e-14, 2.2524e-11, 1.5835e-13, 4.5599e-10, 4.4534e-11, 2.4620e-11,\n 2.3745e-10, 1.2520e-13, 2.4008e-09, 2.2263e-09, 1.2865e-11, 4.6423e-12,\n 1.7901e-10, 1.5474e-13, 3.1398e-11, 1.0631e-10, 1.7146e-12, 5.3075e-12,\n 2.0826e-12, 4.9655e-14, 9.8346e-13, 1.1731e-09, 8.7471e-13, 2.9267e-10,\n 8.7970e-13, 2.3777e-10, 3.1627e-09, 1.3582e-10, 2.2327e-10, 8.4556e-13,\n 1.6882e-12, 2.3763e-12, 1.3015e-13, 3.7742e-10, 1.2610e-12, 2.7395e-13,\n 3.8214e-11, 1.7123e-10, 1.0963e-11, 2.7572e-12, 3.0128e-11, 1.8203e-10,\n 4.5941e-10, 1.0064e-11, 6.4717e-13, 2.3858e-09, 6.2749e-13, 3.5424e-11,\n 2.2162e-13, 2.2603e-11, 9.9094e-12, 3.0581e-11, 2.3805e-12, 1.3752e-10,\n 3.3843e-12, 7.4978e-10, 3.7289e-12, 4.6615e-13, 7.7601e-14, 5.5185e-10,\n 9.5553e-13, 2.7465e-11, 1.8761e-09, 1.7692e-10, 2.5033e-12, 4.5462e-13,\n 1.1097e-11, 1.1315e-13, 7.8743e-13, 8.5199e-11, 6.3427e-13, 1.1623e-11,\n 2.0297e-11, 1.0645e-12, 4.8621e-10, 2.2386e-10, 1.5228e-10, 1.1923e-13,\n 1.8338e-11, 4.1321e-13, 1.3007e-11, 1.0504e-11, 7.1241e-13, 4.6476e-13,\n 1.0090e-09, 1.1245e-12, 2.3377e-13, 7.7149e-11, 1.7913e-10, 8.2115e-11,\n 2.0716e-11, 7.7673e-13, 1.2807e-10, 1.1979e-12, 2.5278e-11, 4.7466e-12,\n 7.4366e-12, 1.9503e-10, 6.0935e-11, 2.0026e-13, 1.1327e-09, 7.4895e-14,\n 5.9083e-13, 1.4675e-12, 3.2873e-10, 6.3185e-10, 1.6045e-13, 2.7671e-12,\n 2.5105e-12, 3.5918e-13, 6.8719e-12, 8.8228e-11, 7.0802e-13, 8.9201e-12,\n 4.0587e-12, 5.1966e-12, 2.2205e-10, 9.2174e-12, 1.2429e-13, 4.0917e-11,\n 4.2176e-11, 6.8730e-12, 1.2611e-12, 3.4185e-12, 8.5571e-12, 1.2270e-13,\n 2.1510e-11, 4.9286e-12, 3.0271e-12, 1.2862e-11, 7.8537e-10, 1.7955e-11,\n 2.1849e-12, 2.9705e-13, 1.1077e-10, 6.4370e-12, 3.8797e-13, 8.4386e-11,\n 1.4013e-10, 2.2554e-12, 9.5224e-14, 2.1626e-10, 4.3869e-13, 6.2717e-11,\n 8.9761e-13, 3.5203e-13, 4.3577e-13, 1.6581e-10, 3.8620e-12, 2.3985e-12,\n 1.0811e-10, 2.9812e-12, 1.0612e-11, 7.6588e-10, 6.7905e-11, 2.3746e-12,\n 1.3411e-11, 3.8488e-12, 1.3383e-11, 7.8202e-11, 8.1933e-11, 2.3134e-11,\n 2.6973e-11, 3.1598e-11, 4.4483e-10, 8.4646e-11, 3.5904e-12, 2.7004e-10,\n 3.8813e-12, 2.2552e-09, 4.0097e-13, 4.1217e-11, 1.4912e-10, 8.6204e-13,\n 1.0211e-12, 3.3973e-10, 2.0074e-10, 7.4310e-13, 1.4905e-10, 4.0237e-13,\n 4.7561e-10, 1.6341e-13, 4.3239e-14, 3.5088e-11, 1.0715e-09, 2.0419e-11,\n 6.0938e-10, 1.3144e-12, 3.3935e-13, 9.0314e-12, 4.0333e-13, 5.3346e-13,\n 6.4395e-13, 1.2313e-12, 1.5754e-12, 3.8346e-13, 6.3448e-10, 9.4511e-11,\n 1.7259e-09, 1.0619e-09, 8.6712e-14, 3.5613e-11, 4.1800e-11, 3.7870e-10,\n 6.7420e-11, 1.7251e-09, 1.7605e-11, 5.6288e-10, 1.0168e-12, 1.0880e-10,\n 6.9705e-12, 2.5444e-10, 1.2557e-10, 1.0314e-13, 1.0975e-09, 9.4736e-12,\n 6.1762e-10, 1.8151e-11, 4.1152e-10, 1.0108e-09], device='cuda:0')" + }, + "57": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-3.8905e-15, -1.8402e-13, 5.2784e-15, -1.3140e-13, -1.4803e-13,\n -3.2763e-15, 8.5978e-15, -2.2486e-13, 4.7237e-15, -2.0174e-14,\n 8.9371e-15, -1.1846e-14, -4.9529e-16, 1.5147e-14, -1.1290e-13,\n -6.4981e-14, 3.0726e-14, -1.1580e-14, -2.4805e-15, -2.7369e-13,\n -8.9529e-14, -3.1361e-13, 2.2459e-15, -1.2457e-13, -8.6357e-15,\n 4.1094e-15, -2.4463e-16, -2.9845e-14, 4.1256e-14, -4.1262e-15,\n 8.3329e-15, 7.3271e-15, -1.0727e-14, 2.3673e-14, 2.0363e-14,\n -2.3623e-15, -7.6771e-14, 8.3494e-16, 6.4332e-16, -2.3549e-14,\n 1.3007e-15, -1.2414e-14, 1.2103e-15, -9.1465e-14, -2.9691e-13,\n 1.4901e-14, -1.1270e-13, 6.0522e-15, -6.7171e-15, -1.7508e-13,\n 8.4729e-17, -3.8291e-15, 2.4496e-14, -1.0552e-14, -3.9075e-15,\n -5.8654e-14, -1.4955e-15, 7.7810e-15, -4.5776e-14, -7.6480e-14,\n -2.0495e-13, -5.3024e-15, -7.4774e-16, 3.7422e-14, -4.3299e-14,\n 3.5890e-15, 1.4559e-14, -1.2857e-14, -9.5420e-16, 7.8301e-15,\n -9.0287e-14, 7.1252e-17, -1.2771e-14, -1.4538e-13, -2.2223e-13,\n -6.9022e-15, -3.1276e-14, 3.1415e-15, -2.2447e-13, -8.4974e-14,\n -1.3853e-15, -2.3793e-13, -1.1533e-16, -9.5576e-14, -2.6874e-15,\n -4.6476e-15, -1.5822e-14, 1.3635e-14, 2.7489e-14, -1.1812e-13,\n -8.5721e-16, -2.0125e-13, 5.4316e-15, -5.7072e-14, 2.7557e-14,\n -4.5175e-14, 1.0075e-15, -1.6623e-14, 1.1014e-14, -9.4610e-15,\n -9.9018e-14, -4.9069e-14, -4.5266e-14, 8.3001e-15, -8.4689e-14,\n -1.4332e-13, 3.0268e-15, -3.0480e-15, 2.1811e-14, -8.7149e-14,\n -2.4029e-13, -6.7861e-14, 1.5435e-14, -2.2470e-15, -2.2867e-14,\n -1.6113e-13, -7.4045e-14, -7.0235e-15, 1.9417e-15, -1.0480e-13,\n -1.6418e-14, -7.3731e-14, -5.0667e-15, -2.8769e-17, -2.1971e-13,\n 6.3803e-15, -1.3203e-13, -6.3427e-14, -8.8219e-16, 1.3331e-15,\n 5.3194e-15, 4.5333e-14, 1.6357e-14, 8.7651e-15, 4.1764e-14,\n -3.1960e-14, -2.0204e-14, -4.9393e-14, 9.2109e-15, 3.9825e-15,\n 2.2004e-15, 1.2090e-14, -8.8098e-15, 1.4164e-15, -7.5566e-15,\n -8.0431e-14, -1.1059e-13, 2.4188e-14, -7.1264e-15, -2.9381e-15,\n -1.1749e-15, 9.0382e-15, 2.2569e-14, -7.7969e-15, 4.4485e-15,\n -1.1690e-14, -1.0049e-13, -2.0706e-14, -6.2290e-15, -3.0514e-17,\n -2.9128e-15, -8.7036e-15, -2.4383e-13, -2.1856e-13, -2.4240e-16,\n -2.5141e-14, -2.3517e-13, -7.8462e-15, -5.8518e-14, -8.0412e-16,\n -1.0153e-13, -3.0200e-14, -6.1680e-14, 5.2711e-15, -9.6924e-14,\n 2.0127e-14, -3.2551e-15, 6.2541e-15, -6.7273e-14, -1.7746e-14,\n -1.1788e-13, 2.7691e-15, -1.6670e-13, -1.2941e-15, -9.1964e-14,\n -3.6733e-14, -2.4808e-13, 1.2906e-14, 1.8810e-15, -6.5469e-15,\n -2.4234e-15, 1.0489e-15, 3.5762e-14, 3.2777e-14, -6.0848e-15,\n -1.1912e-13, 6.5958e-14, -7.3008e-14, -1.3849e-14, -2.4825e-15,\n -1.8088e-13, -7.2946e-14, -4.9254e-16, 2.5872e-14, 1.5480e-14,\n -2.9698e-16, -1.3217e-13, -1.1771e-13, -2.3121e-15, 4.3376e-14,\n -2.6160e-14, -2.2288e-15, -1.2888e-14, -2.6412e-15, -1.9526e-15,\n 4.8756e-16, 4.2387e-15, 1.4562e-14, 9.6998e-15, 9.6653e-15,\n 9.3084e-15, 1.4015e-14, -1.7888e-13, -1.5239e-14, 1.1503e-16,\n -7.3947e-15, -1.6472e-14, -2.4571e-13, -2.5339e-15, -4.3749e-14,\n -2.1526e-14, 8.3327e-15, -9.1239e-14, -9.8692e-14, 4.2462e-15,\n 2.5622e-14, 3.3232e-14, -1.1208e-14, -2.0720e-13, 2.1605e-15,\n 2.0033e-14, -1.6621e-13, -6.5768e-15, -8.6486e-16, -2.3434e-15,\n 3.1535e-15, -4.0770e-14, 2.5002e-14, -7.7804e-15, 9.0327e-15,\n -4.2123e-14, -2.4348e-15, 7.3339e-15, -1.6780e-13, -3.7347e-15,\n 2.2516e-14, -1.5582e-13, -6.1798e-14, -4.6535e-14, 3.4931e-15,\n -1.5567e-13, -4.7781e-14, -1.1516e-13, -6.3901e-15, 2.7640e-14,\n -2.8472e-13, -7.8394e-15, 3.0409e-15, 1.2818e-14, -3.1279e-13,\n -1.2426e-16, 2.2971e-14, 4.7279e-16, 1.1221e-14, 9.0361e-15,\n 9.2787e-15, -1.1370e-13, -9.1008e-14, -4.2962e-15, -9.2054e-15,\n 1.1516e-15, 5.0534e-15, -4.0545e-14, -3.0771e-13, -3.0299e-14,\n -1.9237e-13, -1.1885e-13, 2.1851e-15, -2.4810e-13, -2.3764e-13,\n -2.0623e-13, -2.2167e-13, -1.2174e-14, 1.6101e-14, 4.1744e-14,\n -3.4208e-15, 4.8461e-15, -1.1904e-14, 6.4859e-16, -2.3290e-14,\n -9.2862e-15, -1.9270e-15, 2.2304e-15, -5.2833e-14, -2.4720e-13,\n 1.1851e-15, 1.0277e-15, 4.0811e-14, -1.2760e-13, 3.5341e-16,\n -4.3475e-14, 2.8878e-14, -2.5834e-13, 2.2406e-15, -6.6352e-15,\n 8.0519e-15, 2.7773e-15, -1.3673e-13, -4.4136e-14, 2.4725e-15,\n -1.1323e-13, 2.3145e-15, 2.8880e-14, 6.2352e-16, -5.8466e-14,\n 1.1437e-15, 3.8369e-15, 3.9370e-15, 2.0658e-14, 2.4694e-15,\n -3.2228e-15, -1.6565e-13, -1.5046e-13, -2.7701e-16, -5.2759e-16,\n 3.6495e-15, 2.2692e-14, -4.5984e-14, -2.1039e-13, -4.6251e-14,\n -6.6975e-14, -4.9014e-15, 1.9404e-14, 1.3163e-15, 1.6951e-15,\n -1.1597e-13, -4.8457e-15, 1.0799e-14, -1.1160e-13, -9.0329e-14,\n 3.6005e-15, -6.5879e-16, 9.4880e-15, -1.4344e-14, -4.1348e-15,\n -8.8388e-14, 1.3534e-14, -1.0887e-14, -1.4809e-16, -2.6373e-13,\n -1.2569e-13, -1.4246e-15, -1.5057e-15, 2.2827e-15, 5.0343e-14,\n 5.2716e-14, -3.4261e-15, -1.6426e-15, 1.2934e-14, -2.6290e-13,\n 3.6774e-14, -7.3664e-16, 6.9851e-15, -1.3798e-15, -1.3836e-13,\n -1.9890e-13, 3.2824e-14, -1.3246e-14, -1.4935e-13, 1.9953e-14,\n -1.7499e-14, -2.4571e-15, 4.3078e-15, -9.0986e-14, 1.1325e-14,\n -2.6261e-14, -5.1424e-15, -5.4432e-15, -2.1844e-13, -3.9506e-14,\n -5.6342e-15, 3.3576e-15, 1.6275e-16, -4.2062e-17, -2.4853e-15,\n 3.0042e-15, -1.3934e-15, -1.1679e-13, -3.3728e-13, -1.0672e-13,\n 8.0882e-15, 7.1413e-14, -4.6170e-13, 3.5853e-14, -2.0821e-13,\n 1.0320e-15, -5.4407e-15, -7.6235e-14, -2.4126e-13, -4.1417e-15,\n -2.7199e-13, 3.1310e-14, -4.0798e-14, -5.9532e-15, 4.8866e-14,\n -6.5698e-15, -2.3172e-13, -2.1990e-15, -2.2816e-13, -1.0139e-14,\n 1.1202e-14, 1.4685e-14, 8.3210e-15, -1.3725e-13, -1.0673e-14,\n -1.4450e-14, -1.8099e-13, 8.7674e-15, -2.3089e-13, 2.0623e-14,\n -1.9468e-13, 1.5262e-15, -6.1774e-14, -4.5048e-15, -1.1363e-13,\n -6.7797e-14, 2.9372e-16, -1.0181e-14, -4.8502e-15, 1.3943e-14,\n 7.0957e-16, -2.9924e-14, -1.1817e-14, -2.6109e-13, 4.9414e-15,\n -1.8319e-15, 4.6713e-14, -1.1514e-13, -1.3783e-13, -1.4979e-16,\n -8.9097e-14, 3.4142e-14, 2.2621e-15, -2.3217e-13, 3.4290e-17,\n 1.2931e-14, -2.0485e-13, -4.2287e-14, -2.2453e-15, 5.2701e-15,\n 4.3998e-15, 4.7369e-15, -1.2592e-13, 6.6689e-15, -4.2372e-14,\n -1.7183e-14, -1.1729e-15, -1.8800e-13, 5.1124e-15, -3.4376e-15,\n -3.1961e-16, 5.2910e-15, -1.2185e-13, -1.9546e-13, -1.3081e-14,\n -1.7536e-13, -6.6843e-14, 2.6556e-14, 2.2361e-14, -1.8317e-13,\n 1.0723e-14, -2.0285e-14, 2.3839e-15, -2.5737e-14, -2.7873e-15,\n -2.9646e-15, 1.7588e-14, 1.3402e-14, 6.0448e-15, -1.4656e-13,\n -1.6220e-14, -7.2569e-14, -1.5610e-13, 7.0101e-15, -8.3466e-15,\n -2.8337e-14, -2.5654e-13, -3.9310e-16, 4.5378e-15, -4.7549e-14,\n 2.2914e-14, -3.6357e-15, -8.3074e-16, 4.8730e-15, -6.8650e-14,\n 1.1600e-15, -1.5444e-14, -2.2840e-13, -7.6713e-14, -1.9854e-13,\n -1.0467e-14, 1.5879e-15, -7.4206e-14, -2.4856e-14, 1.4964e-16,\n -9.1661e-15, 4.0188e-15, -5.3585e-14, 2.8511e-14, 3.0872e-16,\n 5.8153e-15, 1.3496e-15, -2.3880e-15, 1.9101e-15, -4.6480e-15,\n -1.5099e-13, 2.3893e-15, -3.3245e-14, -1.0312e-14, 1.0151e-15,\n -1.2058e-13, 8.2753e-15, -1.0608e-15, -2.8004e-14, -3.6962e-14,\n -2.4052e-15, -1.6247e-14, -8.8531e-15, 2.5398e-15, -1.2542e-13,\n -2.2848e-13, -1.5565e-13, -7.3968e-14, -1.3819e-14, 7.2860e-15,\n -4.2441e-15, -3.6133e-14, -5.3266e-15, -9.2494e-14, 3.2833e-15,\n -2.5522e-13, -3.3610e-16, 6.4525e-15, 1.6866e-14, 1.6749e-15,\n -3.1010e-15, 6.9011e-15, -1.7208e-13, -1.0937e-13, 9.1809e-15,\n -2.8081e-15, -1.6909e-13, 2.1123e-14, -1.7602e-15, -1.5358e-14,\n -3.3106e-15, 6.8465e-15, -3.5867e-13, -8.6328e-16, -7.9622e-15,\n -5.0420e-15, -3.6141e-14, -1.1794e-15, -6.0845e-15, -1.5733e-13,\n -1.1674e-14, -7.2395e-15, -2.2240e-14, -1.8593e-13, 2.3415e-14,\n 6.6140e-15, -1.2763e-13, -2.0067e-13, -1.1158e-13, -3.2047e-13,\n -2.0958e-14, 7.5315e-15, -7.4779e-14, -8.8005e-17, -6.4818e-15,\n 2.7663e-14, -1.6781e-13, -2.3620e-13, -6.1224e-16, -1.3617e-15,\n -1.9980e-13, -3.8987e-17, -2.1260e-14, -1.7105e-13, -3.2198e-15,\n -1.7764e-13, 1.6060e-14, -2.3606e-14, -5.5335e-14, -4.9889e-14,\n -7.1777e-14, -3.5357e-13, -3.6098e-14, -3.4224e-14, -4.3443e-15,\n -9.0567e-15, -4.7867e-15, 2.0234e-14, 7.7314e-15, 6.3327e-14,\n 2.4870e-14, -1.4366e-13, -3.0450e-14, -1.9164e-13, -6.6026e-14,\n -1.3305e-14, -1.1930e-13, -2.0505e-13, -1.3126e-15, 1.0104e-13,\n 3.0347e-14, 2.0387e-15, -9.4498e-14, 3.5564e-15, -2.4120e-13,\n 2.5312e-15, -1.1075e-14, -1.6039e-13, -1.2675e-15, -1.0519e-14,\n -3.2910e-15, -4.2626e-14, -1.3751e-13, -2.5093e-13, -2.9473e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.3823e-11, 1.6770e-10, 2.0498e-11, 2.7158e-10, 1.9309e-10, 2.3030e-12,\n 4.6693e-12, 6.2973e-10, 2.8963e-11, 2.5792e-11, 2.9356e-10, 3.1074e-10,\n 4.5566e-10, 2.9079e-13, 2.8093e-11, 1.4578e-11, 4.4516e-12, 1.9983e-11,\n 1.4186e-10, 3.2747e-11, 7.1779e-11, 2.0674e-10, 5.4381e-13, 1.8563e-11,\n 1.7374e-10, 2.4706e-11, 3.5560e-10, 1.3251e-11, 2.8862e-12, 1.4328e-12,\n 9.1676e-12, 3.0460e-12, 6.6629e-12, 6.5887e-12, 1.7674e-12, 8.2331e-12,\n 3.5786e-10, 3.8583e-13, 3.4407e-12, 5.1900e-13, 4.0917e-11, 6.5468e-13,\n 2.5846e-11, 2.3384e-11, 1.1378e-10, 4.0489e-13, 5.3312e-11, 5.0578e-13,\n 1.4597e-12, 3.8500e-10, 7.0374e-12, 2.1298e-10, 2.3345e-13, 4.8850e-10,\n 5.1026e-13, 1.4342e-11, 2.3202e-10, 4.3862e-13, 3.9541e-12, 9.1830e-11,\n 2.4713e-10, 5.6345e-10, 9.8990e-12, 1.6158e-12, 2.3415e-10, 4.8261e-12,\n 5.7109e-11, 4.4526e-10, 3.7259e-11, 1.3794e-09, 5.9033e-11, 3.4221e-10,\n 7.8013e-11, 1.3287e-10, 1.3122e-10, 1.6502e-12, 3.8177e-11, 5.1024e-12,\n 1.0861e-09, 4.0810e-11, 2.8602e-11, 8.9672e-11, 2.7670e-11, 1.3587e-10,\n 1.2835e-12, 1.2959e-12, 4.6961e-10, 2.7874e-12, 5.1425e-13, 6.3728e-11,\n 2.2518e-12, 3.6589e-11, 3.7184e-12, 4.9621e-10, 5.3918e-12, 1.6924e-11,\n 7.1402e-13, 1.9471e-12, 7.5962e-12, 1.7291e-10, 1.4792e-10, 8.1460e-11,\n 3.3800e-10, 5.1670e-13, 1.5956e-10, 2.6499e-10, 4.0149e-11, 3.0399e-10,\n 3.0170e-13, 4.9353e-11, 7.6080e-10, 1.4695e-10, 1.9989e-09, 3.7990e-10,\n 1.1314e-11, 5.5170e-10, 1.0198e-10, 6.2405e-10, 2.3079e-12, 6.0989e-11,\n 5.4140e-11, 1.8249e-11, 3.7913e-13, 7.1249e-12, 3.2535e-10, 7.7425e-11,\n 1.5547e-10, 1.1589e-10, 1.4569e-11, 4.3272e-12, 1.0939e-12, 2.2231e-10,\n 5.9679e-10, 3.0600e-11, 3.9102e-10, 5.1337e-12, 4.2384e-10, 5.1296e-11,\n 5.5718e-12, 3.2452e-11, 1.3231e-11, 8.2706e-11, 1.6225e-10, 1.9167e-11,\n 1.2318e-10, 1.7726e-10, 1.2167e-10, 1.1894e-11, 2.9558e-11, 1.2688e-11,\n 6.2723e-13, 2.3159e-13, 3.7376e-10, 3.0699e-10, 2.6539e-11, 7.2658e-11,\n 1.5729e-10, 1.4405e-11, 2.7722e-11, 3.8852e-11, 2.5769e-11, 1.3826e-12,\n 1.7493e-09, 6.8930e-10, 3.6841e-12, 6.6239e-11, 4.5065e-11, 1.8009e-12,\n 6.5196e-11, 2.7652e-10, 4.1627e-11, 5.3733e-11, 1.8213e-10, 2.0321e-11,\n 1.4439e-10, 1.1787e-10, 1.2543e-11, 2.4335e-10, 7.9790e-11, 1.3643e-10,\n 4.2414e-10, 6.2962e-11, 1.0727e-10, 5.0628e-11, 2.0760e-11, 1.5818e-10,\n 1.3138e-09, 1.1424e-10, 9.4755e-13, 1.8674e-10, 1.8550e-12, 1.2101e-12,\n 5.5756e-12, 6.8162e-12, 1.1344e-10, 1.9463e-10, 1.2054e-11, 1.0198e-10,\n 7.0885e-10, 7.7991e-12, 3.4953e-11, 1.1978e-10, 7.9877e-10, 5.9421e-12,\n 2.0095e-12, 3.6579e-11, 6.8948e-11, 8.2328e-12, 9.3965e-10, 3.5714e-11,\n 6.5292e-12, 3.5757e-11, 4.2024e-12, 4.0416e-12, 8.7332e-13, 4.2900e-11,\n 1.9755e-12, 6.3695e-13, 9.1343e-13, 6.6401e-12, 3.0616e-13, 4.1450e-12,\n 3.8108e-11, 1.4765e-11, 1.0181e-12, 3.4382e-13, 1.2891e-11, 8.0203e-10,\n 8.2433e-12, 5.0404e-11, 2.0685e-10, 1.9936e-11, 3.6656e-10, 5.6634e-11,\n 5.7886e-10, 1.4500e-10, 9.6549e-12, 1.9024e-10, 3.1379e-10, 1.3538e-12,\n 6.5498e-13, 1.1330e-10, 5.1706e-10, 1.5818e-12, 1.3228e-10, 4.4356e-11,\n 9.3524e-11, 5.7254e-12, 1.8465e-10, 4.0980e-11, 5.1088e-10, 3.7910e-10,\n 7.6110e-12, 5.3651e-10, 1.1780e-10, 3.4704e-11, 8.4269e-11, 1.1819e-10,\n 3.7487e-12, 4.8759e-12, 1.5082e-10, 5.2325e-12, 1.7096e-10, 1.3396e-12,\n 5.2013e-12, 1.7633e-10, 1.7711e-10, 6.7689e-12, 8.6220e-12, 9.7585e-10,\n 9.7178e-13, 4.9708e-13, 1.6738e-11, 2.7000e-12, 3.6629e-11, 2.2540e-13,\n 5.8924e-11, 8.9939e-11, 3.6966e-10, 6.3875e-12, 4.2175e-12, 1.2922e-11,\n 4.7848e-10, 7.0354e-10, 1.9791e-11, 1.6621e-10, 5.6544e-11, 1.8729e-11,\n 4.1968e-10, 3.0188e-10, 1.9971e-10, 1.7968e-10, 2.2992e-10, 1.1639e-12,\n 1.6167e-11, 5.9550e-11, 4.9408e-12, 1.8536e-10, 1.2975e-10, 5.3105e-12,\n 5.6536e-10, 2.9666e-11, 9.2947e-12, 5.8612e-10, 1.7379e-10, 7.8017e-12,\n 1.5192e-12, 2.1008e-10, 1.3640e-10, 4.6571e-12, 2.3106e-10, 1.5691e-12,\n 1.0735e-09, 5.5086e-11, 6.0648e-10, 7.9809e-11, 1.9465e-12, 7.8674e-11,\n 2.5009e-10, 6.3446e-12, 1.9078e-11, 4.7623e-11, 1.5088e-12, 5.4388e-11,\n 3.7492e-10, 5.4400e-13, 7.5361e-13, 9.7667e-11, 1.1920e-12, 1.1784e-10,\n 3.4373e-13, 1.1757e-10, 3.3449e-11, 5.6749e-12, 6.8594e-12, 1.1593e-11,\n 2.8439e-10, 1.7839e-12, 1.4878e-10, 8.1061e-11, 1.5307e-10, 2.6058e-12,\n 1.7818e-11, 1.2542e-12, 2.4997e-11, 7.9066e-11, 7.4482e-11, 1.4429e-10,\n 4.0404e-11, 4.6267e-11, 1.2192e-12, 6.0406e-13, 5.5330e-13, 1.2432e-10,\n 1.4305e-12, 2.9933e-10, 5.2872e-12, 7.1023e-12, 7.8742e-11, 1.2504e-09,\n 1.3788e-11, 9.7361e-13, 5.7191e-12, 1.0738e-12, 3.0818e-11, 1.5308e-12,\n 3.8894e-11, 1.6929e-10, 4.2781e-10, 3.8563e-10, 1.3125e-10, 5.1415e-11,\n 1.7888e-12, 2.2632e-11, 3.1169e-11, 5.9822e-11, 3.0853e-11, 1.0142e-10,\n 1.9143e-10, 1.2493e-11, 4.0699e-11, 1.0081e-11, 4.0910e-13, 1.9130e-11,\n 8.5163e-13, 7.5247e-10, 2.1170e-11, 1.4521e-11, 7.9130e-10, 8.1251e-11,\n 1.6970e-10, 3.5018e-12, 1.7400e-09, 2.7777e-10, 2.2665e-12, 2.8326e-10,\n 2.5813e-11, 1.1541e-10, 9.7160e-10, 1.8349e-11, 4.3288e-11, 1.6297e-10,\n 3.3010e-09, 2.8619e-10, 4.8664e-11, 9.0664e-13, 3.7559e-11, 6.1116e-11,\n 2.0886e-10, 6.1556e-12, 7.3174e-11, 1.1641e-12, 1.5183e-10, 5.4916e-11,\n 1.6894e-12, 6.1970e-13, 7.1403e-10, 1.9801e-11, 2.9746e-10, 7.2852e-11,\n 1.2094e-12, 9.1925e-12, 1.0874e-12, 1.7981e-10, 1.3405e-10, 3.9643e-11,\n 1.4900e-10, 2.4972e-13, 1.4683e-09, 1.6329e-09, 6.1703e-11, 1.6120e-10,\n 8.6230e-11, 1.4263e-12, 2.9031e-11, 1.3430e-10, 3.8449e-11, 2.4066e-12,\n 8.1802e-13, 6.3799e-13, 2.3544e-11, 4.8447e-10, 1.1154e-11, 1.0713e-10,\n 1.3330e-11, 3.6717e-10, 1.7231e-09, 1.1168e-10, 1.1540e-10, 1.6055e-12,\n 2.3996e-12, 1.2242e-12, 2.4736e-12, 1.9231e-10, 3.3228e-11, 2.2225e-12,\n 3.4344e-11, 1.6941e-10, 4.4066e-12, 1.0847e-11, 2.9233e-10, 2.6888e-10,\n 1.4804e-10, 3.2929e-12, 1.6928e-11, 7.6886e-10, 1.6326e-11, 5.1969e-11,\n 2.0466e-12, 3.3460e-11, 4.0407e-12, 3.1921e-10, 8.3809e-12, 1.2009e-10,\n 7.3426e-12, 5.1878e-10, 1.0425e-11, 6.6621e-12, 5.0679e-13, 4.5785e-10,\n 7.5954e-13, 3.7515e-11, 1.8310e-09, 1.0409e-10, 1.1142e-12, 1.2489e-11,\n 5.1809e-12, 7.0260e-13, 1.5122e-12, 1.6361e-10, 1.4692e-11, 1.2603e-10,\n 4.1802e-11, 4.0551e-12, 7.3388e-10, 1.0804e-10, 9.4199e-11, 2.0472e-12,\n 7.8212e-12, 2.5352e-12, 6.2895e-12, 3.2944e-11, 2.7407e-12, 9.3871e-12,\n 4.0891e-10, 1.7146e-11, 2.5981e-12, 3.0298e-10, 1.6234e-10, 9.6091e-11,\n 8.1487e-11, 1.3969e-11, 5.8252e-11, 1.1103e-11, 4.6587e-10, 1.2622e-11,\n 3.8884e-11, 8.9602e-11, 3.4847e-11, 7.0814e-12, 5.9162e-10, 1.9013e-12,\n 5.2587e-13, 1.0512e-11, 2.7450e-10, 3.0679e-10, 1.4117e-12, 2.1620e-11,\n 1.6674e-11, 6.3296e-12, 8.9782e-11, 3.7608e-11, 1.6004e-11, 1.0335e-10,\n 3.3988e-11, 2.9082e-12, 6.6473e-11, 2.9111e-10, 1.0879e-12, 7.0111e-11,\n 4.7650e-11, 1.2828e-10, 1.0643e-11, 1.9031e-11, 3.1119e-12, 1.3403e-13,\n 5.0908e-11, 1.7061e-10, 1.3201e-11, 8.6983e-11, 1.2819e-09, 7.8859e-12,\n 1.0801e-12, 9.6206e-13, 4.6218e-10, 2.1950e-10, 2.9925e-12, 1.7257e-10,\n 1.4239e-10, 3.1831e-11, 2.2768e-12, 5.9755e-10, 2.0891e-12, 7.7651e-11,\n 1.2650e-11, 2.9374e-12, 2.3431e-12, 9.1597e-10, 9.9848e-11, 6.9714e-11,\n 2.0715e-10, 3.6867e-11, 5.3617e-12, 4.9981e-10, 1.3408e-10, 9.3112e-12,\n 9.9542e-11, 3.6455e-11, 2.9946e-11, 2.5428e-11, 6.3204e-11, 3.9454e-11,\n 1.9492e-10, 1.3485e-10, 3.2417e-10, 8.3432e-11, 1.3730e-12, 2.0615e-10,\n 3.3552e-11, 6.6613e-10, 3.2205e-12, 4.1094e-11, 1.3093e-10, 3.3287e-11,\n 6.3297e-12, 3.7263e-10, 2.7016e-10, 1.4748e-11, 7.5369e-11, 1.5654e-13,\n 8.8955e-10, 4.2027e-12, 6.5908e-13, 6.1623e-11, 8.4891e-10, 1.5425e-11,\n 5.8419e-10, 8.3159e-13, 2.9953e-13, 1.1429e-11, 4.3712e-12, 1.4804e-11,\n 2.3575e-12, 9.3418e-12, 5.4824e-12, 1.3749e-12, 4.1326e-10, 5.2993e-11,\n 3.0464e-10, 5.3587e-10, 6.2421e-13, 3.7750e-11, 1.0965e-10, 1.3516e-10,\n 2.5560e-11, 1.6685e-09, 7.6118e-12, 4.9615e-10, 2.0665e-11, 5.6974e-10,\n 7.7736e-11, 4.8897e-10, 8.4684e-11, 6.3691e-13, 7.9992e-10, 5.2086e-11,\n 1.6665e-10, 1.7370e-10, 1.5380e-10, 1.6444e-09], device='cuda:0')" + }, + "58": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-6.2421e-17, 1.7080e-17, -5.6052e-45, ..., -1.1228e-15,\n -2.4989e-14, -5.6052e-45],\n [ 1.9927e-16, 1.2560e-16, -5.6052e-45, ..., 1.4107e-15,\n 5.4064e-14, 5.6052e-45],\n [-2.4825e-17, -2.9797e-17, -5.6052e-45, ..., -1.0529e-16,\n -9.4722e-15, 5.6052e-45],\n ...,\n [-7.5219e-17, 1.1056e-18, 5.6052e-45, ..., -4.6129e-15,\n -2.6368e-14, -5.6052e-45],\n [-1.2815e-16, 2.9256e-16, -5.6052e-45, ..., -3.7798e-15,\n -4.4758e-14, -5.6052e-45],\n [ 8.3428e-17, 7.9064e-17, 5.6052e-45, ..., 1.1642e-15,\n -1.6698e-15, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3196e-10, 1.0066e-10, 9.6705e-15, ..., 1.2959e-10, 3.0995e-11,\n 3.1750e-12],\n [6.2895e-12, 5.1472e-12, 1.4457e-15, ..., 5.5822e-12, 8.2094e-12,\n 1.4609e-13],\n [1.2115e-10, 1.4999e-10, 4.8672e-16, ..., 1.7833e-10, 4.4184e-11,\n 4.0971e-13],\n ...,\n [1.2648e-12, 9.8768e-13, 6.0638e-17, ..., 1.4881e-12, 4.5481e-13,\n 1.4900e-14],\n [9.0616e-13, 6.0260e-13, 1.7391e-16, ..., 1.2928e-12, 3.7812e-13,\n 5.2004e-15],\n [1.7561e-11, 1.4098e-11, 2.4889e-16, ..., 1.7984e-11, 7.8307e-12,\n 1.1365e-15]], device='cuda:0')" + }, + "59": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-4.6619e-12, 3.7357e-12, -4.1890e-13, 4.7121e-13, -1.0578e-12,\n -1.9417e-12, -1.5126e-12, -1.1338e-11, -3.0178e-12, 9.5566e-14,\n 2.0477e-12, -4.9462e-14, 1.2402e-12, -1.3586e-12, 8.7406e-13,\n -3.2983e-13, -2.9723e-12, -1.3561e-13, -4.2013e-12, -2.9781e-12,\n 3.7331e-14, -1.2798e-12, -2.0571e-13, -2.3155e-12, -2.9631e-13,\n -2.9735e-13, 3.9261e-12, -7.2444e-15, 2.1928e-12, -9.8889e-13,\n -4.9728e-12, -1.5436e-12, 1.1065e-12, -5.4541e-12, 5.4131e-14,\n 4.7318e-12, -1.5728e-12, -4.3007e-14, 3.5606e-12, -1.3370e-12,\n 1.6467e-12, -2.9924e-12, 7.9303e-14, 6.7220e-13, 1.6301e-13,\n -2.3066e-12, -1.1296e-11, 2.8338e-12, 3.9672e-12, -5.6705e-12,\n -5.4995e-12, 4.9816e-14, 2.3911e-13, -5.0638e-14, 5.2902e-13,\n 2.6700e-13, 2.8903e-12, 6.1585e-13, 3.1911e-12, 2.0647e-12,\n 3.3483e-12, 9.8504e-14, 4.8490e-12, -2.9114e-13, 3.0358e-12,\n 2.5675e-13, -5.0524e-13, 3.8383e-13, -1.4058e-12, 3.3720e-12,\n -7.7321e-12, -3.6703e-12, 2.5501e-12, 1.3803e-12, -9.8266e-12,\n 2.3930e-12, -3.6966e-12, 6.1735e-13, 5.6411e-12, -7.1732e-13,\n -4.8818e-12, -4.2687e-12, -3.9494e-12, -4.7252e-12, -1.9303e-12,\n 3.9654e-12, 2.6283e-12, 1.1027e-12, 6.4552e-12, 4.7421e-12,\n 7.7862e-13, -1.1857e-12, -1.5336e-12, -1.6897e-12, 1.5156e-12,\n 3.2104e-12, -1.7823e-12, -1.0300e-11, 4.3204e-12, 3.5059e-12,\n 1.0853e-12, -9.1558e-13, 5.7395e-12, 6.8715e-13, -5.4118e-14,\n -3.2455e-12, 1.4921e-13, -2.3631e-12, 1.0298e-12, -3.7377e-12,\n 2.0903e-12, 2.0499e-12, 1.4124e-13, 5.7465e-12, -7.0590e-12,\n 5.1559e-12, -2.3366e-12, -2.7007e-13, 1.9117e-12, 6.1656e-12,\n -7.4360e-13, 3.4746e-13, -1.3984e-11, -1.2473e-13, -3.4051e-12,\n 1.5277e-12, -1.9499e-13, -3.3322e-13, -3.3581e-13, 2.1607e-12,\n -5.0857e-13, -1.4846e-13, -2.9632e-13, 1.0430e-12, -1.3336e-12,\n 8.1872e-13, 1.8185e-13, -8.6300e-12, 8.8696e-13, 1.6811e-14,\n -1.2060e-12, -4.2521e-13, 4.4655e-13, 6.9822e-13, -1.0280e-11,\n 1.7405e-12, -2.0237e-12, 7.7792e-13, -5.3140e-13, -3.5870e-13,\n 1.0371e-12, 4.0983e-13, 6.5773e-13, -5.0204e-13, -1.7020e-12,\n 7.6229e-13, 9.8125e-13, 3.0939e-13, -2.2570e-12, -2.8452e-12,\n 1.2342e-12, -2.3621e-12, 2.2201e-12, 7.0164e-13, -4.1338e-13,\n 4.7706e-14, -5.3834e-13, 2.7244e-12, -3.5168e-12, -2.8740e-13,\n -5.3947e-13, 4.1311e-12, 2.3606e-12, -2.4309e-12, -3.6993e-12,\n -5.2131e-13, -5.2311e-12, 1.8924e-12, 1.5480e-12, -1.8939e-13,\n -1.3376e-11, 2.0615e-12, 1.0399e-13, 6.4839e-13, 8.3529e-13,\n 4.2489e-14, -5.5694e-13, -4.3516e-12, 1.8381e-13, 2.1330e-12,\n -1.4246e-12, 1.3659e-13, 2.9250e-13, 3.6486e-12, 2.4056e-13,\n -3.8454e-12, 2.1591e-12, -3.3374e-12, 7.2265e-13, 3.6612e-12,\n 2.9945e-12, -1.5833e-12, -4.1015e-13, -4.8751e-12, 1.0654e-12,\n -2.1346e-13, -2.9883e-13, 1.3562e-12, 2.2096e-12, -2.5486e-12,\n -2.7105e-12, 1.8945e-12, 4.8714e-13, 7.9345e-13, 1.3278e-12,\n 3.3483e-12, -3.0675e-13, -3.9941e-13, 3.1472e-12, -6.7919e-14,\n 4.3386e-13, 8.7902e-14, 6.3866e-13, -1.6717e-12, -3.2538e-12,\n -8.6241e-13, -1.4797e-11, -3.9086e-12, 4.1540e-12, 4.0929e-13,\n 2.1798e-12, -1.6323e-12, 1.6543e-12, -5.3159e-13, 4.9890e-12,\n 2.1938e-12, 6.1041e-12, -1.4074e-12, 4.1128e-12, -2.3459e-12,\n 5.2919e-12, 2.1278e-13, -3.0074e-13, -3.1635e-13, -1.7647e-12,\n 2.6810e-13, 8.9192e-13, 1.0824e-12, 3.1507e-12, 1.1812e-12,\n -4.1671e-12, -1.2177e-12, -2.3285e-12, 1.1930e-12, 5.4596e-12,\n 1.4385e-13, -8.4996e-14, -1.4911e-12, 2.4940e-12, -7.6504e-12,\n 2.1784e-12, 2.4544e-12, 9.5958e-13, 5.3769e-13, -2.4058e-12,\n 5.1698e-14, 7.2029e-13, -3.6545e-14, 2.5573e-13, -1.0830e-12,\n -2.0319e-12, 4.5628e-12, 8.6122e-13, 1.0349e-12, -2.6430e-12,\n -1.1789e-12, 5.2769e-12, 2.8301e-13, -2.3149e-12, 5.0180e-13,\n -5.1938e-12, -3.3840e-12, 1.3969e-11, -5.5164e-12, -2.5399e-12,\n 2.1864e-12, 6.7278e-13, 5.8615e-12, -4.3064e-12, -5.0644e-13,\n -5.3061e-12, -2.6885e-12, 2.9949e-12, 2.2435e-13, -4.8549e-13,\n 4.3736e-12, 6.6855e-14, 1.1294e-12, 4.0082e-12, -4.6634e-12,\n 1.7541e-13, -2.7186e-13, -1.7709e-12, 5.2531e-12, 2.6983e-12,\n -2.4404e-12, 1.7291e-12, 5.8958e-13, -1.5805e-12, 1.5193e-12,\n 3.5828e-12, 6.1140e-14, 5.4150e-13, -1.4083e-12, 5.8749e-12,\n 3.4053e-12, 2.0717e-12, -6.1911e-12, -3.3119e-12, 9.7196e-14,\n 1.4319e-12, 7.2917e-13, 2.0948e-12, -2.9255e-14, -6.0635e-13,\n -3.9407e-12, -5.2609e-13, -3.5167e-13, 2.2187e-12, -3.7859e-12,\n -1.2557e-12, 2.5297e-12, -3.9601e-12, 1.1033e-13, 4.9220e-12,\n -1.0827e-12, 1.0165e-12, 1.2278e-13, 5.6412e-13, -3.6958e-12,\n -4.0288e-12, -3.1979e-12, 1.4038e-12, -1.5030e-12, 7.5041e-12,\n 2.2385e-12, -5.6354e-12, 1.6355e-12, 1.7203e-12, -5.5431e-12,\n 4.1989e-12, 6.4141e-13, 2.2811e-13, 2.3347e-12, 5.3268e-14,\n 3.0397e-12, 1.0909e-12, 1.6967e-12, 2.2937e-12, -1.0008e-12,\n 2.3482e-13, 5.5506e-12, -1.1715e-12, 4.0158e-12, 2.8405e-12,\n -2.1280e-13, 1.3929e-12, -5.0234e-13, 1.5744e-12, 2.9187e-12,\n 1.0576e-12, 4.1754e-13, 1.1219e-12, -2.0183e-11, 8.3940e-13,\n 1.0543e-12, -9.3898e-13, -6.8996e-13, -3.4223e-12, 2.6510e-12,\n 1.9283e-13, 4.2627e-13, 2.3838e-13, -6.7456e-13, -1.3869e-12,\n 4.0099e-13, -2.4213e-12, 1.8045e-14, 9.9153e-13, -4.8930e-12,\n 1.7800e-12, -2.9456e-12, 3.3609e-12, -4.5209e-13, -4.3049e-13,\n 1.8401e-12, 1.2352e-12, 1.8970e-12, -1.2829e-12, -5.1805e-13,\n 5.2972e-12, 2.2176e-12, -5.2285e-12, -1.3692e-12, -4.6124e-12,\n 3.4418e-13, -2.0746e-12, -2.2930e-12, -1.9691e-12, 1.0059e-12,\n -3.8728e-12, 5.4599e-12, 7.7712e-12, -6.4249e-13, 6.1368e-12,\n 4.9259e-13, 3.7706e-13, -3.3182e-12, -5.4457e-12, 4.7608e-12,\n -3.8471e-12, 1.3326e-12, 5.8009e-12, 5.6394e-12, 2.5862e-12,\n 2.4243e-14, 1.3768e-13, 1.3040e-12, -1.6136e-13, 1.1347e-12,\n -1.7124e-14, -1.4597e-13, 2.4262e-13, 4.9447e-13, 2.7379e-12,\n 2.0326e-12, -4.8400e-13, -1.3206e-11, 1.7740e-12, 1.8892e-13,\n 2.9732e-12, -9.5639e-13, 1.3810e-12, 2.5262e-13, -3.1035e-12,\n -2.4796e-13, 6.0404e-13, -6.0413e-13, 1.7573e-14, -3.1594e-12,\n 1.4369e-13, 6.0310e-12, -9.7712e-13, 3.4011e-12, -3.9590e-12,\n 4.3680e-13, -4.6044e-13, -2.7304e-13, 2.3449e-12, 3.3040e-12,\n 3.3863e-12, 3.6753e-13, 1.6178e-12, -2.1531e-12, 1.3096e-12,\n 7.1260e-13, 7.4169e-13, 6.3091e-12, 9.1213e-13, -1.7861e-12,\n 3.0871e-13, -3.2563e-12, 2.0544e-12, 2.0792e-12, 1.1762e-12,\n 8.1792e-13, 6.2947e-13, 1.1692e-12, 4.4856e-12, 1.4837e-12,\n 1.8798e-12, -1.3409e-13, 9.4404e-13, 4.9667e-13, 2.2291e-14,\n -1.3949e-12, -2.2493e-12, 5.9634e-13, 1.3032e-12, 1.6384e-12,\n -1.5735e-12, -5.2045e-12, 4.0259e-13, 3.4910e-12, 8.7288e-13,\n 1.2159e-12, -7.5877e-12, 7.9958e-12, 8.4317e-12, -6.3466e-13,\n 4.1362e-12, -7.2914e-12, 2.2203e-13, 3.1149e-12, 4.1296e-13,\n -2.9854e-12, -3.9238e-13, -2.6158e-12, 1.2499e-12, -1.4012e-12,\n 9.7239e-12, 5.8479e-13, 6.9118e-13, -1.6580e-11, 4.1325e-12,\n 1.9966e-12, 3.8363e-12, 1.9011e-13, 9.9005e-13, 1.2054e-12,\n 2.0677e-12, -3.7278e-14, 1.0669e-12, 2.9438e-13, 7.2740e-13,\n 3.9069e-13, 1.0198e-12, -1.2481e-11, 3.6684e-13, 9.3193e-13,\n 9.3260e-13, -2.9816e-13, 1.0215e-12, 2.2400e-13, -2.5125e-13,\n 2.1884e-12, -6.3226e-13, -1.6356e-12, -1.5819e-12, 2.6272e-13,\n -6.6423e-13, 1.1547e-13, -7.5011e-12, -4.7842e-13, -2.6272e-12,\n -2.8306e-12, -5.6563e-13, 6.2641e-12, 2.2650e-12, 8.1237e-13,\n -4.8849e-13, 3.2133e-13, -2.6944e-12, 3.1843e-12, 8.2806e-13,\n -7.2019e-12, 1.3663e-12, 3.5121e-12, 5.2339e-12, -2.9672e-14,\n 1.2787e-14, -9.1497e-12, 6.1777e-13, -1.0758e-11, -4.2294e-13,\n 4.8359e-12, 2.2652e-12, -6.2445e-13, -2.7937e-12, 4.1569e-13,\n 1.8850e-12, -8.1778e-12, 2.8668e-12, 2.3405e-12, 3.6286e-12,\n 7.4149e-12, 8.9115e-13, 1.1735e-13, -3.3054e-13, 5.4763e-13,\n 2.1710e-12, 7.6847e-13, 1.1280e-12, 7.0344e-12, -3.2933e-12,\n 1.1546e-14, 3.4506e-12, 6.4295e-12, -4.8462e-14, 2.7339e-13,\n 1.6539e-12, -7.7357e-14, -8.1377e-12, 1.6127e-12, 1.1717e-12,\n 5.1761e-12, -4.5799e-15, 3.2251e-13, 3.3302e-12, -7.3770e-14,\n -1.0586e-12, 3.9219e-12, -4.7167e-13, -5.2040e-12, 5.9835e-13,\n 4.4882e-13, -5.8616e-12, -7.5769e-12, -5.9686e-12, -8.7034e-13,\n 7.1536e-13, -9.1089e-13, 1.2341e-12, 1.2521e-14, 9.8488e-13,\n 2.3070e-12, -1.0649e-12, 4.8090e-12, 2.9030e-13, 2.3813e-12,\n -4.8011e-13, 5.1900e-12, 3.2349e-13, 1.9351e-12, 6.7904e-13,\n -1.4206e-12, 1.0743e-12, -9.8561e-13, 4.7345e-13, -5.5863e-12,\n 2.3207e-13, 9.1714e-13, 3.0945e-12, 7.5402e-14, 5.4666e-13,\n -1.4359e-12, 3.8547e-12, -1.9667e-12, -2.1194e-12, 1.2880e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([4.0905e-06, 4.4958e-07, 6.4828e-06, 3.4485e-08, 3.2537e-07, 8.7494e-08,\n 2.8450e-07, 2.5492e-05, 7.9818e-06, 9.1131e-07, 2.1039e-08, 4.3855e-06,\n 1.4021e-07, 2.9979e-08, 3.6344e-07, 1.9453e-08, 8.5819e-08, 2.7360e-05,\n 1.2905e-05, 1.2546e-06, 4.4437e-07, 1.7358e-06, 9.0383e-07, 2.5718e-06,\n 3.2335e-06, 2.0772e-07, 4.6254e-08, 4.7358e-08, 1.6107e-07, 6.5102e-07,\n 2.1571e-05, 1.7306e-06, 4.5115e-06, 1.4671e-05, 3.9799e-07, 2.2094e-08,\n 1.2445e-07, 9.6592e-07, 1.4937e-08, 5.8549e-07, 2.7519e-08, 5.4936e-06,\n 9.3833e-07, 4.0746e-08, 6.4161e-07, 2.2000e-08, 7.4367e-06, 9.9396e-08,\n 3.4041e-08, 9.5749e-06, 2.0854e-06, 7.4481e-06, 4.5734e-08, 7.9085e-07,\n 1.0516e-06, 2.1932e-06, 4.3849e-07, 5.0929e-08, 9.7416e-08, 2.2617e-06,\n 1.0400e-06, 6.4835e-07, 1.0179e-07, 2.6318e-08, 3.3193e-06, 1.5390e-06,\n 5.1182e-08, 1.1220e-05, 1.4258e-06, 2.3566e-05, 3.8801e-06, 6.9508e-06,\n 1.0282e-06, 2.1075e-06, 8.2065e-07, 1.2044e-08, 7.1284e-06, 3.3853e-06,\n 2.7464e-05, 3.5912e-06, 4.8810e-07, 8.8916e-08, 4.7829e-06, 1.5821e-05,\n 8.3767e-06, 8.3114e-07, 7.7546e-08, 2.0262e-06, 1.3498e-07, 9.8376e-09,\n 4.3022e-06, 2.3620e-07, 2.1791e-06, 3.0815e-06, 7.2166e-07, 1.2110e-05,\n 9.9208e-07, 1.2721e-05, 1.2173e-07, 3.0304e-06, 2.9691e-07, 1.4935e-08,\n 1.0317e-07, 3.5281e-07, 5.7156e-08, 6.8734e-06, 2.1991e-06, 4.5648e-06,\n 1.3987e-08, 2.3446e-08, 8.5002e-06, 1.9394e-06, 1.1256e-05, 8.5530e-08,\n 1.0748e-06, 2.2543e-08, 4.7124e-07, 3.4878e-05, 4.1375e-06, 1.2899e-06,\n 1.2914e-07, 7.3116e-07, 1.7598e-06, 7.3239e-07, 6.7380e-06, 3.2310e-08,\n 1.5265e-07, 7.0410e-06, 6.1367e-08, 2.6028e-08, 2.4473e-07, 4.2330e-07,\n 8.8785e-06, 3.4580e-07, 3.1121e-06, 1.9807e-05, 2.7439e-08, 3.5016e-06,\n 1.0043e-05, 2.4105e-08, 1.7721e-06, 4.6069e-06, 2.7543e-08, 2.8805e-07,\n 4.1037e-07, 2.5078e-08, 4.9698e-07, 3.3983e-06, 1.1310e-06, 7.5871e-06,\n 4.2969e-08, 1.2665e-07, 5.1913e-06, 1.2607e-06, 2.3244e-07, 1.2517e-07,\n 1.2107e-07, 1.2610e-07, 2.6135e-06, 1.4789e-06, 1.0984e-05, 1.5537e-07,\n 3.2663e-05, 7.4246e-07, 1.3609e-07, 9.0467e-06, 5.3737e-07, 2.6968e-07,\n 1.3796e-05, 2.4277e-06, 4.9364e-06, 1.4208e-07, 1.3789e-07, 5.4962e-06,\n 1.3259e-06, 1.9334e-05, 8.8529e-07, 7.4110e-06, 1.6325e-08, 2.9991e-08,\n 3.1841e-06, 2.0042e-06, 9.4974e-07, 1.3457e-06, 1.1882e-08, 1.1172e-06,\n 1.5085e-05, 1.5206e-05, 1.9429e-07, 6.9816e-06, 5.4287e-06, 1.3218e-07,\n 1.2853e-07, 2.6670e-08, 1.1931e-05, 2.9261e-06, 1.6326e-08, 2.4684e-06,\n 6.0623e-06, 5.0073e-07, 6.4316e-08, 5.6018e-08, 8.5227e-07, 1.4074e-05,\n 5.9463e-07, 4.1187e-06, 1.2110e-07, 6.1605e-08, 2.2295e-07, 2.2584e-06,\n 3.2294e-06, 2.5192e-08, 4.5400e-06, 5.1993e-06, 3.8096e-07, 6.3951e-07,\n 1.4440e-06, 1.9880e-06, 1.0607e-06, 3.1375e-05, 1.4564e-06, 6.8258e-08,\n 3.4777e-06, 9.7080e-09, 4.7094e-08, 2.8284e-08, 2.1947e-05, 3.9097e-06,\n 1.1800e-07, 1.3683e-07, 4.1755e-08, 2.1837e-08, 1.1344e-05, 6.0109e-07,\n 2.3091e-07, 8.4260e-08, 1.3573e-08, 2.0656e-05, 3.0269e-07, 6.2046e-07,\n 2.7521e-07, 1.0455e-08, 1.6127e-06, 1.2271e-07, 2.4523e-08, 1.1224e-05,\n 6.4824e-06, 7.8706e-06, 2.3248e-06, 3.1604e-07, 1.0375e-05, 1.2327e-06,\n 3.9263e-06, 2.6235e-07, 2.8819e-08, 8.8170e-06, 6.9111e-06, 2.7071e-07,\n 2.8935e-06, 5.6540e-06, 3.4825e-08, 1.4541e-08, 1.7024e-07, 3.8746e-06,\n 1.1665e-05, 6.4661e-07, 7.5201e-07, 1.2815e-07, 3.1772e-06, 2.2548e-07,\n 9.1428e-08, 7.0706e-07, 1.4441e-06, 1.5046e-06, 3.0499e-06, 5.0469e-08,\n 4.6602e-07, 2.7740e-07, 1.1684e-05, 3.8591e-08, 1.8447e-06, 1.1160e-05,\n 3.7330e-07, 1.6117e-05, 3.7633e-08, 3.2497e-06, 1.7097e-07, 1.8919e-08,\n 3.7178e-05, 1.6735e-07, 1.8615e-05, 5.1338e-06, 1.7050e-07, 5.9781e-07,\n 1.9677e-08, 4.7838e-07, 1.1612e-07, 1.6831e-06, 4.2002e-07, 2.3934e-06,\n 3.1086e-06, 2.1124e-06, 5.3977e-07, 8.8727e-08, 5.4662e-07, 5.0229e-06,\n 3.1254e-07, 2.5652e-07, 2.4744e-06, 1.1276e-08, 2.3860e-06, 1.1644e-07,\n 4.5010e-06, 1.8572e-06, 3.5147e-08, 1.2506e-06, 5.5121e-08, 7.1517e-06,\n 5.5401e-08, 8.8189e-06, 1.6504e-06, 1.5795e-07, 1.4955e-08, 6.0393e-08,\n 9.6322e-07, 1.1219e-06, 7.0535e-07, 1.1504e-06, 4.4653e-07, 9.0536e-06,\n 1.5669e-07, 1.6921e-08, 1.0258e-05, 6.5960e-08, 4.6932e-08, 3.0831e-06,\n 7.5263e-06, 9.2008e-07, 1.6301e-06, 1.3056e-08, 8.1065e-08, 1.1512e-06,\n 2.6199e-08, 3.4423e-06, 1.4326e-07, 3.6179e-07, 4.3162e-06, 1.5634e-05,\n 5.0687e-08, 2.1341e-05, 3.3444e-08, 2.6098e-07, 1.3084e-06, 1.9760e-07,\n 3.1135e-08, 2.7945e-07, 1.8275e-06, 2.9640e-08, 1.9996e-08, 3.2281e-06,\n 1.2739e-06, 2.6410e-07, 1.0987e-07, 1.5458e-07, 2.4912e-07, 3.9076e-08,\n 2.5072e-08, 5.5522e-06, 7.4182e-07, 1.5914e-05, 1.3377e-05, 1.7808e-06,\n 2.7200e-07, 5.6368e-06, 2.9957e-06, 5.6884e-08, 1.9563e-06, 5.8323e-08,\n 5.1554e-08, 1.0966e-07, 2.1776e-07, 8.1065e-06, 2.0108e-06, 2.7857e-07,\n 2.1875e-06, 1.5942e-05, 5.2271e-06, 8.4678e-08, 3.4763e-07, 1.1504e-05,\n 4.1458e-06, 2.3256e-06, 1.9714e-07, 1.1808e-05, 2.8186e-08, 2.7773e-08,\n 1.3141e-05, 5.6333e-07, 1.1167e-05, 1.1388e-05, 5.3388e-08, 6.9046e-08,\n 1.1072e-06, 5.0339e-06, 8.7636e-06, 9.9951e-07, 2.1814e-06, 1.0250e-06,\n 2.7307e-06, 1.8475e-06, 7.9099e-07, 6.7076e-08, 2.2339e-07, 1.1364e-06,\n 5.4372e-07, 3.4669e-06, 6.3730e-07, 1.4197e-05, 5.5386e-07, 5.5411e-06,\n 1.3310e-06, 1.1056e-06, 2.4784e-07, 5.6744e-07, 3.5685e-06, 1.8809e-06,\n 2.8950e-06, 5.2738e-08, 8.8504e-07, 4.5252e-08, 2.7042e-06, 4.9043e-06,\n 8.9106e-07, 5.0466e-07, 5.2859e-07, 1.0609e-06, 4.7645e-06, 2.4835e-06,\n 1.6866e-08, 2.6696e-08, 5.5194e-07, 1.2348e-07, 6.5615e-07, 2.9471e-06,\n 4.8392e-06, 9.8286e-07, 2.4503e-06, 1.3938e-06, 7.5582e-08, 6.8503e-07,\n 1.8013e-07, 3.8079e-08, 3.1806e-07, 2.3217e-06, 8.9889e-06, 4.1387e-06,\n 1.3854e-07, 1.1308e-07, 8.8996e-07, 1.3457e-06, 1.2046e-07, 4.9758e-06,\n 2.6439e-08, 1.7078e-06, 1.4740e-06, 9.5771e-06, 2.2333e-06, 1.0044e-06,\n 2.9426e-08, 1.8881e-07, 4.4354e-06, 3.2702e-05, 1.5823e-06, 5.1148e-08,\n 1.3396e-05, 6.3279e-06, 2.7609e-08, 2.0033e-08, 1.5219e-07, 5.3906e-08,\n 6.8698e-08, 1.2553e-07, 1.8167e-05, 4.0816e-08, 1.0908e-07, 1.2989e-06,\n 2.2718e-06, 9.3111e-09, 2.9962e-07, 1.5043e-06, 4.3995e-08, 5.0312e-06,\n 1.1947e-08, 3.5150e-08, 2.0462e-07, 1.0819e-05, 1.9143e-07, 2.2334e-08,\n 2.4004e-06, 3.8509e-06, 1.2378e-07, 6.2642e-06, 1.3022e-06, 1.6288e-07,\n 1.0916e-06, 1.4716e-07, 2.3044e-06, 2.4962e-06, 2.3994e-08, 4.1292e-07,\n 4.9892e-08, 1.3723e-07, 5.9516e-06, 8.5027e-06, 4.0380e-06, 1.4041e-06,\n 5.0353e-08, 8.0167e-07, 2.1677e-05, 2.0362e-08, 5.3675e-06, 2.4306e-07,\n 2.3190e-08, 2.0110e-06, 4.2934e-08, 4.4239e-07, 1.5872e-08, 7.0946e-06,\n 3.7962e-07, 1.6841e-08, 5.9868e-07, 2.5441e-05, 9.8611e-06, 8.1013e-07,\n 6.3688e-08, 1.7034e-06, 4.3224e-07, 4.4449e-05, 3.1007e-06, 4.3657e-07,\n 1.6574e-06, 4.2598e-06, 9.0242e-06, 1.8790e-08, 1.7665e-06, 1.1278e-06,\n 2.5642e-06, 1.2035e-06, 9.3275e-08, 7.5212e-06, 1.2381e-05, 3.9925e-07,\n 3.6777e-06, 9.4886e-09, 3.9755e-05, 2.0885e-05, 1.3540e-06, 4.8216e-06,\n 1.9673e-08, 5.1114e-06, 1.1500e-06, 1.1727e-05, 1.7039e-07, 6.6455e-06,\n 4.9152e-06, 5.3191e-08, 7.1708e-08, 1.0661e-05, 1.2454e-05, 1.6380e-06,\n 3.3276e-07, 7.3796e-06, 1.0684e-05, 1.1249e-06, 3.7246e-06, 1.8077e-07,\n 2.7573e-06, 1.1021e-07, 9.5197e-07, 2.0903e-07, 4.9286e-07, 8.2705e-08,\n 6.9340e-07, 4.5080e-07, 1.2047e-07, 2.3964e-07, 1.9008e-08, 6.8244e-08,\n 8.4169e-06, 5.7528e-07, 1.0897e-07, 3.2489e-07, 9.3690e-06, 1.7132e-07,\n 2.6238e-08, 1.9072e-08, 3.3530e-07, 5.8272e-06, 2.1044e-08, 4.6445e-07,\n 2.8854e-07, 7.2133e-08, 2.0347e-07, 1.4240e-06, 4.8734e-06, 6.8875e-08,\n 7.3837e-06, 5.6127e-06, 3.8965e-06, 2.3853e-06, 3.8830e-06, 5.8396e-08,\n 1.5010e-08, 5.2409e-06, 2.6583e-06, 2.7414e-07, 1.1889e-07, 1.8177e-07,\n 1.4011e-06, 1.5810e-07, 1.2920e-06, 3.0364e-06, 4.1610e-08, 1.4575e-08,\n 2.2938e-05, 4.1327e-06, 8.2531e-06, 2.0837e-05, 1.7268e-06, 1.1296e-05,\n 1.9669e-05, 1.3373e-07, 7.0754e-08, 2.9376e-08, 4.4718e-07, 1.2783e-07,\n 2.3541e-08, 4.3983e-08, 3.4217e-08, 5.0195e-07], device='cuda:0')" + }, + "60": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-8.8247e-14, -3.4930e-14, 4.4694e-15, 2.0168e-15, 3.1353e-15,\n 2.5236e-14, 7.0442e-14, -5.4691e-13, -1.0911e-14, -3.0371e-15,\n 9.4409e-15, -2.0276e-16, -1.8181e-15, 6.4097e-15, -2.6588e-13,\n -1.4456e-15, -1.2670e-14, 8.3605e-15, -6.8716e-14, -5.6897e-13,\n 1.4341e-15, -1.4867e-13, -7.0125e-15, 3.0337e-14, 6.5763e-15,\n 7.4489e-15, 4.3320e-14, -9.2437e-15, -3.7108e-14, -3.6193e-14,\n -6.6862e-14, -3.6448e-14, -2.6482e-17, -3.4965e-13, 1.6722e-15,\n 3.7459e-14, 2.1946e-15, -7.9270e-15, -2.7217e-17, -1.1593e-14,\n 2.1596e-15, 5.6144e-16, -2.3492e-15, -1.7069e-15, 6.9853e-15,\n 3.1327e-14, -3.0474e-13, 3.5073e-15, 5.8377e-15, -2.5140e-13,\n -9.7109e-14, -2.9861e-14, -6.4291e-16, -2.1490e-14, -1.6510e-14,\n -5.0089e-15, -4.6799e-15, -9.4420e-17, 1.4480e-14, 8.8924e-17,\n -1.8120e-13, -8.1106e-15, -1.3177e-15, 6.6195e-15, -1.5408e-16,\n -1.1422e-14, 5.6415e-15, -5.9995e-15, -4.0454e-14, 3.7696e-15,\n -5.2141e-13, -1.1741e-14, 3.2858e-15, 1.0191e-15, -4.1129e-13,\n 5.5671e-15, -2.7993e-13, -1.4880e-16, -1.5579e-14, 1.6607e-14,\n -6.3995e-14, -1.3943e-13, -5.4886e-14, -1.0410e-13, -2.5207e-14,\n -4.0708e-15, -1.3375e-14, 1.7753e-15, 5.8680e-15, 4.7290e-14,\n 1.1687e-15, -5.7906e-13, -1.9942e-14, -1.8649e-14, -2.2615e-14,\n -4.9402e-15, -2.4745e-13, -1.6860e-13, 2.2699e-14, -8.0383e-15,\n -6.3628e-15, -3.9636e-14, 5.1100e-14, 1.3889e-15, 2.5027e-15,\n -8.4688e-14, -8.7722e-16, -1.4305e-13, 2.7849e-15, -4.4558e-15,\n -2.3317e-15, -5.5786e-15, 1.9725e-16, 3.3606e-14, -1.3525e-13,\n 2.2008e-14, -5.2314e-13, 2.2463e-15, -8.8935e-15, 4.2330e-14,\n 6.4146e-15, 4.5884e-15, -5.2524e-13, 1.2574e-15, -1.5561e-13,\n 1.6334e-15, 3.1507e-15, -1.2062e-13, 1.7394e-15, -7.0650e-15,\n 1.3255e-15, 6.6075e-15, 1.9223e-15, 3.8178e-15, 1.5968e-14,\n -2.9028e-14, 6.3386e-16, -1.0167e-12, -5.8056e-14, 1.1700e-14,\n 2.1514e-15, -7.3235e-15, 6.7687e-15, -2.1166e-15, -6.7391e-13,\n -3.7833e-15, -7.5490e-14, 5.8397e-16, 1.8161e-15, -1.4003e-15,\n -2.5557e-15, 1.9885e-15, 7.7434e-16, 6.6215e-16, 4.9155e-15,\n 4.6002e-16, 3.1928e-17, -1.4011e-15, -2.0913e-13, -6.7276e-14,\n 1.7782e-14, 1.9560e-15, -6.5052e-14, 2.4413e-15, -4.0174e-14,\n 7.6920e-16, 7.5024e-15, 3.1330e-14, -8.9369e-14, 2.6920e-15,\n -1.9823e-14, -2.2344e-14, 6.5930e-16, -4.7098e-14, -4.4165e-13,\n 1.9595e-15, -1.2683e-13, -7.2812e-15, -1.6124e-15, -1.1384e-14,\n -6.3630e-13, 2.2629e-15, 1.8377e-15, 7.1963e-15, 1.3259e-15,\n 8.2398e-15, -2.3628e-14, -1.5901e-14, 2.3187e-15, 2.0762e-14,\n 1.6858e-14, 8.9399e-16, 1.3891e-15, 8.9510e-16, 2.1981e-15,\n -7.4663e-14, 1.6075e-16, -1.3178e-13, 7.3494e-16, 1.3227e-14,\n -3.9542e-15, -2.1908e-13, 2.4143e-15, -2.4137e-13, -1.7220e-14,\n -1.2497e-14, -1.9731e-15, -8.3290e-15, -9.2011e-15, 3.1596e-14,\n -3.5547e-14, -2.6978e-15, -4.7811e-15, -1.2852e-15, -3.2065e-17,\n 1.9659e-14, -9.4973e-15, -6.6296e-15, -4.0220e-16, -8.9343e-16,\n 1.3754e-15, -7.3847e-17, -3.4375e-14, 3.1334e-15, -1.7378e-13,\n -5.1965e-15, -5.0936e-13, -5.4530e-13, -3.7171e-14, 1.5653e-15,\n 7.1066e-15, 1.1208e-14, -6.9056e-15, -1.8366e-14, -9.5523e-15,\n -1.4048e-14, 5.7019e-14, -2.2817e-14, 1.9172e-14, 2.1357e-14,\n 2.1318e-14, 4.3617e-15, 7.4235e-15, -2.1400e-15, 7.6916e-15,\n 1.1668e-14, -6.0938e-14, 1.9999e-15, -2.0007e-13, 1.3172e-15,\n -1.2609e-13, -7.8220e-15, -3.0925e-14, -8.7393e-16, 8.8620e-14,\n -8.2854e-16, 9.7954e-15, -9.2160e-14, -1.9439e-14, -2.2902e-13,\n -9.3689e-15, -9.2113e-15, -2.5529e-15, -4.5411e-15, -8.6764e-14,\n -1.4178e-13, 1.4218e-15, 2.8228e-15, -1.3540e-15, 4.0214e-15,\n 4.4080e-14, 1.6001e-14, 1.8189e-15, -2.6144e-15, 7.0521e-15,\n -1.3510e-14, -5.8167e-13, 2.1151e-15, -4.1659e-14, -5.9004e-15,\n -4.6655e-13, -1.3034e-14, 2.5192e-13, -1.2500e-13, 2.1356e-14,\n -7.9552e-14, 1.6719e-15, 9.7484e-14, -3.7031e-13, 1.0883e-14,\n -1.6227e-13, -8.2794e-14, -1.1852e-14, -5.2792e-15, 5.5183e-15,\n -3.9783e-16, -7.7527e-15, -4.3641e-13, -3.7138e-15, -2.3172e-13,\n 3.0486e-15, -1.4691e-13, 3.7985e-16, -1.9538e-15, -7.7039e-14,\n 4.1788e-14, -5.6951e-16, -1.6232e-15, -3.2789e-14, -6.2139e-15,\n 2.1612e-14, -7.9334e-16, 2.8632e-15, 1.0900e-14, 1.0459e-14,\n -9.4619e-14, -1.4008e-14, -1.7072e-13, -2.1637e-15, -1.9130e-15,\n -1.6593e-15, -2.0124e-15, -5.7210e-15, -1.1798e-14, -1.2164e-14,\n -5.1311e-13, -1.0508e-14, -3.8566e-14, -7.2836e-15, -1.3056e-13,\n 1.3564e-14, 2.7062e-15, -1.7039e-13, -4.4387e-15, 2.7919e-14,\n 6.9096e-15, -4.6926e-16, 6.2728e-15, -4.0546e-13, -7.8856e-14,\n -1.4003e-13, 8.2311e-15, -2.6522e-15, -2.9652e-14, -2.6477e-14,\n -8.6932e-15, -1.8636e-13, 2.6494e-14, 3.2572e-15, -4.5747e-14,\n -2.5385e-14, -1.6314e-15, -2.0635e-15, -8.9076e-15, -6.5893e-15,\n -3.3335e-14, 2.3153e-16, -1.6060e-15, 3.2383e-15, 9.4684e-15,\n -7.5863e-16, 3.4636e-14, 1.1362e-14, -4.7819e-14, 1.3604e-14,\n 6.8639e-16, -1.7500e-15, 3.5998e-15, -1.2894e-14, 1.2958e-15,\n 3.5840e-15, -3.0128e-15, -7.2568e-16, -8.4528e-13, 1.6807e-14,\n -4.5568e-15, -4.6846e-16, 4.0398e-15, -1.1861e-14, -2.3714e-14,\n 1.3454e-14, 1.2085e-14, 2.2626e-15, -2.3458e-13, -2.1214e-13,\n 6.3649e-15, -3.5267e-13, -1.6058e-15, -5.3856e-15, -1.5194e-13,\n 1.9211e-17, -8.4164e-14, -1.4308e-14, 1.0350e-14, 9.4936e-15,\n -2.9880e-15, -1.3629e-13, -1.9976e-14, -1.9218e-13, -5.5739e-15,\n 4.3664e-15, 3.1137e-15, -3.7142e-14, 9.0707e-16, -2.7917e-13,\n 2.6763e-15, -2.4405e-13, -2.6092e-13, -3.3889e-13, -4.2718e-15,\n -5.4865e-13, 3.6417e-14, 4.3750e-14, 5.4163e-15, 3.7853e-14,\n 5.9254e-15, -1.4681e-14, -1.0548e-13, -1.3276e-13, 2.0287e-14,\n -6.7114e-14, -7.8934e-15, 7.9145e-14, -6.2058e-14, -7.2694e-15,\n 8.3432e-15, 1.1432e-15, -9.6337e-16, -1.7073e-14, 7.5816e-17,\n 1.8074e-14, 1.9700e-15, -9.2192e-16, -4.1220e-15, -5.4444e-15,\n -2.1727e-13, -1.2233e-15, -6.1949e-13, -3.9680e-18, 9.2169e-16,\n -3.7890e-13, 4.0826e-15, -1.3897e-14, 1.1123e-14, 8.1105e-15,\n 9.5339e-15, -1.6360e-14, 2.5140e-15, 8.1938e-15, 4.3987e-15,\n -5.5985e-15, 5.5184e-14, -3.1280e-15, -1.9774e-15, 1.7317e-14,\n -1.0191e-15, 3.5313e-15, -6.3023e-15, 2.1812e-15, 3.8993e-15,\n 3.0359e-15, -1.1736e-15, -3.7705e-15, -5.4855e-14, -1.5548e-15,\n 3.5809e-15, 2.3108e-16, -4.9664e-14, -4.3784e-15, -1.0479e-13,\n -3.3282e-14, 6.8364e-15, 4.5342e-15, -1.5147e-15, -3.0840e-16,\n -7.2213e-15, 3.4304e-15, -1.7026e-14, 3.0324e-14, -7.7960e-15,\n 1.2573e-14, 1.1724e-14, -6.0788e-16, -1.0416e-15, 8.2949e-15,\n -9.0624e-15, -1.8700e-14, -1.0884e-15, -6.1409e-14, -1.0539e-12,\n 4.3229e-15, -5.6248e-13, -1.5065e-15, -1.5458e-14, -1.7077e-15,\n -8.6531e-14, -7.2330e-14, 6.5069e-14, -1.2190e-15, -9.7469e-15,\n 2.4565e-14, -2.5426e-13, -4.3130e-15, 5.5848e-15, -1.2069e-15,\n -5.0029e-13, -2.4193e-14, -4.8390e-13, 2.1753e-15, 1.0771e-15,\n 1.3342e-13, -2.6730e-15, 2.7269e-15, -4.9790e-13, -6.8682e-15,\n -1.6409e-15, 1.7477e-14, -3.0538e-15, -6.9341e-15, -1.2296e-16,\n -1.7364e-14, 1.1090e-15, -2.8609e-16, 2.2267e-15, -1.5150e-15,\n 2.2344e-15, -3.6484e-15, -3.3799e-13, 3.6441e-15, 2.3211e-14,\n 3.5337e-15, 1.9220e-15, 3.6023e-15, -3.6255e-15, -3.6493e-15,\n 1.4075e-15, -9.7515e-16, 1.9497e-14, 2.8986e-14, -4.8800e-13,\n -1.0085e-12, 3.4798e-15, -1.5638e-13, -2.9706e-16, -1.8074e-14,\n -8.5887e-13, -1.0044e-14, 3.9070e-14, -9.4197e-15, -3.1962e-16,\n 7.3137e-15, 1.8545e-16, -9.3347e-14, -7.5060e-15, 1.0395e-15,\n -2.5471e-14, 5.4853e-15, -6.7513e-15, 5.3597e-14, 8.6996e-15,\n -1.4599e-14, -2.0648e-13, -3.5689e-16, -6.6091e-13, -2.8588e-15,\n 3.4408e-14, 2.0652e-15, -5.8587e-14, -8.2626e-14, -5.3214e-15,\n -5.7488e-15, -1.0383e-13, -4.9576e-15, -2.4366e-15, -2.7538e-14,\n 6.7128e-14, 8.8299e-16, -4.2453e-14, 4.6153e-15, -3.7529e-14,\n -2.1511e-14, -1.4566e-13, -6.3857e-15, 1.8958e-13, 1.7602e-15,\n 1.3167e-15, 1.1372e-14, 8.0598e-14, -5.6865e-15, -1.6968e-17,\n -6.8704e-15, -4.6171e-15, -2.7625e-13, 5.7301e-15, 2.2527e-16,\n -1.2690e-15, -1.4019e-14, 4.8206e-15, 1.3959e-14, 2.9137e-15,\n -1.4412e-13, 2.3558e-14, 9.4479e-17, -6.7491e-14, 5.1908e-15,\n -1.0184e-15, -4.9404e-13, -2.2680e-13, -1.3829e-13, -1.0419e-13,\n 3.3412e-14, 4.9538e-15, 5.3854e-16, 1.0534e-14, -7.2737e-15,\n -4.8677e-14, -1.2934e-13, 4.2006e-15, -2.0083e-15, 1.7526e-14,\n 7.0764e-15, 2.5790e-14, 4.6557e-15, -2.0626e-15, -1.0667e-14,\n 1.8672e-14, -4.6620e-15, -2.0153e-14, 1.8621e-14, -2.8805e-13,\n 2.8243e-15, -4.2319e-15, -7.8570e-15, 4.6070e-17, 7.6088e-16,\n 7.1214e-15, 1.5412e-14, -1.3546e-14, -2.3745e-14, -7.7382e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.2511e-10, 5.4863e-11, 2.1067e-09, 6.6115e-12, 1.0967e-10, 1.1271e-12,\n 1.0308e-11, 1.5570e-08, 5.3879e-09, 1.4876e-10, 5.2429e-12, 9.7442e-11,\n 3.8982e-11, 4.4509e-13, 3.4313e-11, 3.7908e-13, 7.8579e-12, 4.7223e-09,\n 8.2750e-10, 3.3829e-10, 2.6377e-10, 8.7624e-10, 1.8631e-10, 1.2821e-10,\n 1.3282e-10, 2.7341e-11, 5.6269e-13, 5.2081e-13, 1.9910e-12, 8.0375e-12,\n 5.2179e-09, 9.3641e-11, 8.6073e-10, 5.7651e-10, 6.4425e-11, 1.1490e-13,\n 1.1796e-11, 3.6655e-10, 4.7248e-13, 6.4756e-11, 7.1425e-13, 1.5335e-09,\n 1.9463e-10, 8.5179e-13, 8.6227e-11, 3.7499e-12, 1.0538e-09, 8.6650e-13,\n 7.9222e-13, 7.3386e-10, 2.9142e-11, 8.4256e-10, 9.0230e-13, 1.5937e-11,\n 1.5964e-10, 2.4559e-10, 2.1623e-12, 5.2313e-12, 2.0095e-11, 1.1820e-10,\n 6.8165e-11, 1.1756e-11, 2.9854e-11, 5.9917e-13, 1.0146e-10, 1.3882e-10,\n 8.6324e-12, 1.0390e-09, 1.6902e-11, 3.1930e-09, 5.0063e-10, 2.2672e-09,\n 7.8166e-12, 9.6732e-11, 3.7675e-11, 3.7434e-13, 2.3894e-09, 7.5417e-11,\n 3.9106e-09, 9.6877e-10, 2.2824e-10, 2.1509e-12, 3.8555e-09, 3.8012e-09,\n 2.0839e-10, 4.5738e-12, 1.6271e-11, 7.4732e-11, 3.7049e-12, 1.4798e-13,\n 1.4669e-09, 4.1806e-11, 2.9971e-11, 3.0154e-10, 1.0058e-11, 2.8023e-09,\n 1.0436e-10, 3.7432e-09, 5.1020e-12, 4.5404e-11, 1.7885e-12, 1.5688e-12,\n 1.1423e-12, 1.6150e-12, 6.4631e-13, 8.7840e-10, 3.2840e-10, 1.7143e-10,\n 5.9299e-13, 5.9669e-12, 6.6518e-10, 6.6403e-11, 2.3292e-10, 1.7820e-12,\n 9.9052e-10, 6.8476e-12, 1.3539e-10, 9.8366e-09, 1.0433e-09, 5.4854e-11,\n 9.3796e-13, 4.6116e-12, 4.0162e-10, 5.6547e-11, 2.4048e-09, 5.7826e-12,\n 2.4179e-12, 2.8059e-09, 6.5116e-13, 4.0925e-12, 5.7024e-12, 2.9108e-12,\n 1.8198e-10, 4.2647e-12, 1.5656e-11, 1.7308e-09, 4.4051e-12, 2.0795e-09,\n 2.5759e-09, 2.2907e-12, 8.2259e-11, 1.4067e-10, 3.8153e-12, 2.7446e-11,\n 8.5402e-11, 3.0940e-12, 2.9279e-10, 2.6160e-11, 1.3784e-11, 1.9178e-09,\n 8.3375e-12, 3.0079e-12, 6.1945e-11, 1.3198e-11, 2.6574e-11, 3.8350e-12,\n 1.6579e-11, 1.3595e-11, 5.1210e-10, 8.5029e-10, 3.3025e-09, 6.2567e-11,\n 1.2269e-08, 4.9549e-12, 5.6655e-11, 1.2594e-09, 9.9129e-11, 8.7724e-12,\n 2.9881e-09, 6.5693e-10, 3.5563e-09, 5.7903e-12, 5.6095e-12, 9.0521e-10,\n 1.1743e-10, 6.0536e-10, 9.0533e-11, 1.1912e-09, 3.4418e-12, 8.6978e-13,\n 1.2387e-09, 4.1802e-11, 4.0932e-12, 3.1454e-10, 1.5181e-12, 2.4290e-10,\n 1.0278e-09, 4.6323e-09, 1.1699e-12, 1.1008e-09, 1.2644e-10, 1.8814e-12,\n 1.2998e-12, 6.1844e-13, 1.6798e-09, 4.4124e-11, 6.1996e-13, 1.0196e-10,\n 1.8850e-10, 4.4327e-12, 5.4128e-12, 1.4073e-12, 1.4214e-11, 5.4209e-10,\n 9.9310e-12, 7.3468e-10, 5.3289e-11, 1.2334e-12, 2.4586e-12, 1.9722e-11,\n 6.5977e-10, 5.3641e-12, 5.6800e-10, 5.7891e-10, 8.6226e-12, 6.5887e-12,\n 9.6249e-11, 2.0248e-10, 6.7124e-11, 4.8799e-09, 3.2568e-11, 1.5983e-11,\n 1.6504e-10, 1.5042e-12, 1.6109e-12, 4.9819e-13, 6.4887e-09, 1.4820e-09,\n 1.8340e-12, 9.6113e-13, 2.2632e-12, 4.1554e-12, 6.0365e-09, 6.3164e-12,\n 4.0951e-12, 2.4794e-11, 4.7068e-13, 7.9295e-09, 7.3401e-12, 5.0648e-10,\n 3.1128e-12, 2.9037e-13, 4.1389e-11, 5.8837e-13, 8.7467e-13, 3.5658e-09,\n 1.8548e-09, 1.2842e-10, 1.0641e-09, 3.4452e-11, 5.0828e-09, 1.8575e-10,\n 9.4416e-10, 7.1064e-12, 2.1017e-12, 4.3490e-10, 3.1478e-10, 6.3807e-11,\n 4.3237e-11, 1.8008e-09, 4.4616e-13, 4.6733e-13, 5.0834e-12, 1.2554e-09,\n 4.1014e-09, 1.4352e-11, 1.1502e-11, 2.5284e-12, 5.7948e-11, 8.8935e-11,\n 7.5047e-13, 5.1216e-12, 6.3700e-11, 2.4080e-11, 2.1350e-10, 1.7355e-11,\n 2.6982e-10, 2.7605e-11, 8.2450e-09, 2.0415e-12, 2.1255e-10, 5.0097e-09,\n 1.0189e-11, 5.5967e-10, 2.4896e-12, 1.4906e-10, 2.7331e-12, 6.9143e-12,\n 9.4301e-09, 2.0752e-12, 9.6963e-10, 3.0385e-10, 5.3921e-11, 8.6695e-12,\n 1.1441e-12, 3.5327e-12, 4.7901e-11, 8.2358e-10, 3.0049e-10, 3.0112e-10,\n 9.4223e-10, 9.8320e-10, 1.4579e-11, 1.3962e-11, 7.4776e-11, 9.6736e-10,\n 1.5787e-12, 9.2674e-11, 1.8399e-10, 2.9339e-13, 4.1657e-11, 3.6538e-11,\n 6.8973e-11, 8.9708e-11, 6.9086e-12, 1.4872e-10, 3.1343e-11, 3.2256e-09,\n 1.5045e-11, 2.4190e-10, 2.4732e-11, 1.2955e-12, 3.5974e-13, 1.3733e-11,\n 5.4900e-11, 1.8619e-10, 5.9481e-12, 1.7652e-11, 1.8987e-12, 5.4617e-10,\n 5.4652e-12, 1.4687e-12, 1.7604e-09, 4.4705e-12, 1.9054e-11, 1.9172e-09,\n 7.4295e-10, 1.6210e-10, 8.8620e-10, 4.5670e-13, 5.9361e-12, 4.3749e-10,\n 6.6399e-12, 1.0038e-10, 5.2702e-11, 3.9291e-12, 1.8964e-09, 1.3579e-09,\n 4.7639e-13, 5.8119e-09, 1.6357e-12, 4.0231e-12, 8.4325e-10, 2.8117e-12,\n 1.0047e-11, 1.9043e-11, 2.0407e-11, 1.1295e-12, 3.6340e-13, 4.7359e-10,\n 5.9253e-11, 1.5708e-11, 3.1592e-12, 7.9755e-12, 1.8941e-12, 8.9964e-12,\n 3.3784e-12, 3.2883e-09, 2.5857e-11, 7.3102e-10, 3.7783e-09, 3.0689e-11,\n 3.1299e-12, 2.3472e-09, 3.2075e-11, 2.4331e-11, 1.2275e-10, 1.1352e-12,\n 1.1377e-11, 5.3101e-11, 2.6678e-12, 2.5983e-09, 6.7875e-10, 5.1575e-11,\n 1.0331e-10, 3.8484e-09, 2.7996e-10, 1.8001e-12, 2.0489e-12, 2.7495e-09,\n 7.5296e-10, 2.3873e-10, 8.3863e-11, 5.1707e-09, 8.2657e-13, 1.1247e-12,\n 5.5143e-09, 6.7068e-12, 1.7789e-10, 3.2280e-10, 3.7577e-12, 2.1597e-11,\n 8.3123e-12, 6.0200e-11, 2.7532e-10, 9.7451e-12, 4.2678e-10, 5.8160e-11,\n 3.0482e-10, 3.0467e-11, 1.0379e-10, 3.6933e-13, 6.5284e-11, 3.3328e-10,\n 9.0493e-12, 1.9983e-11, 2.0538e-12, 9.1543e-09, 3.6721e-11, 8.0809e-11,\n 3.2684e-10, 5.1046e-12, 3.9307e-12, 9.7010e-11, 4.1322e-11, 4.8885e-10,\n 7.4199e-11, 4.9430e-13, 4.0317e-12, 1.6696e-12, 1.8979e-10, 3.4499e-10,\n 7.2499e-12, 5.7278e-12, 1.1715e-11, 8.4220e-11, 1.8846e-09, 1.4767e-10,\n 7.4619e-13, 1.9915e-13, 1.9533e-10, 3.2718e-12, 1.6474e-11, 9.6347e-11,\n 1.0050e-09, 6.8269e-11, 2.1102e-11, 2.6167e-11, 6.7270e-12, 1.7265e-10,\n 6.4869e-12, 5.4594e-13, 4.9637e-11, 3.7522e-11, 5.3022e-09, 8.5719e-10,\n 6.9667e-12, 1.7901e-11, 4.5500e-11, 2.6412e-11, 2.6255e-12, 1.6448e-10,\n 6.4067e-12, 1.6742e-11, 8.4418e-12, 2.5983e-09, 1.1577e-10, 2.6243e-10,\n 4.9032e-12, 8.8462e-12, 3.6399e-09, 1.4826e-08, 2.0714e-11, 9.2984e-13,\n 1.8858e-10, 8.9030e-11, 4.5420e-12, 2.7648e-12, 1.0650e-12, 4.3719e-13,\n 1.8334e-12, 5.4736e-12, 2.2767e-09, 4.8905e-13, 1.8100e-11, 1.7607e-11,\n 1.9700e-09, 1.4589e-12, 4.7697e-12, 9.5833e-10, 3.8977e-11, 2.9403e-10,\n 5.6304e-13, 3.4820e-12, 7.8801e-11, 2.6949e-09, 2.5472e-11, 4.9281e-13,\n 1.4879e-11, 1.8556e-10, 1.4785e-12, 1.0638e-09, 7.0141e-11, 6.1217e-12,\n 5.6874e-12, 1.5592e-11, 4.5754e-11, 4.8096e-10, 6.5100e-12, 1.1434e-11,\n 1.8509e-12, 3.6935e-11, 1.3274e-09, 2.7705e-09, 5.2909e-11, 2.7558e-11,\n 9.7576e-13, 3.4706e-11, 1.1446e-09, 4.3564e-12, 1.3066e-10, 1.6414e-12,\n 1.3147e-12, 1.2053e-09, 1.4991e-12, 2.1328e-12, 1.2798e-12, 2.7203e-09,\n 3.4286e-12, 1.9203e-12, 7.5457e-12, 7.7158e-10, 2.5944e-09, 5.5741e-10,\n 8.3437e-13, 1.4629e-11, 8.7517e-11, 1.6431e-08, 3.3560e-10, 1.2936e-10,\n 1.6846e-09, 5.6023e-11, 3.7391e-09, 1.6777e-12, 2.0541e-11, 7.5010e-10,\n 3.8654e-10, 1.7103e-11, 1.3705e-12, 2.0359e-10, 1.6721e-10, 3.1869e-12,\n 1.9786e-09, 1.2761e-13, 4.4803e-09, 3.1949e-09, 1.5171e-11, 7.3519e-10,\n 2.2773e-12, 1.7951e-10, 5.5783e-10, 4.0467e-09, 2.2218e-12, 3.6479e-09,\n 9.0195e-11, 2.7963e-12, 4.4424e-12, 2.2515e-10, 2.0920e-09, 3.2606e-11,\n 6.2111e-12, 1.6602e-09, 5.3097e-10, 1.4703e-11, 2.2319e-10, 4.3739e-12,\n 1.2790e-10, 6.3542e-12, 1.3948e-11, 1.0677e-10, 2.6380e-12, 3.3688e-12,\n 2.4146e-12, 3.1294e-12, 4.0683e-12, 2.8099e-11, 5.8275e-12, 1.2769e-12,\n 3.9798e-09, 1.6342e-10, 5.0828e-11, 2.7693e-12, 4.5251e-09, 2.1893e-12,\n 3.3556e-13, 1.3984e-12, 2.0875e-10, 1.1876e-09, 3.5618e-13, 3.0573e-11,\n 2.3668e-11, 1.0256e-12, 1.2580e-11, 5.7210e-10, 7.1191e-10, 1.0785e-12,\n 1.4236e-09, 2.7714e-09, 1.0780e-09, 3.1381e-11, 7.4152e-10, 2.1328e-12,\n 2.8663e-13, 3.0765e-10, 9.8392e-12, 1.0997e-11, 8.4595e-13, 2.1888e-12,\n 9.6761e-12, 2.3142e-12, 1.6098e-11, 1.3436e-10, 3.2470e-13, 1.0350e-12,\n 1.0101e-09, 4.3072e-11, 7.9559e-10, 2.1873e-09, 7.8994e-11, 3.8486e-10,\n 5.5849e-09, 3.1712e-12, 6.3329e-12, 9.8373e-13, 5.6581e-12, 1.1651e-11,\n 5.8056e-12, 1.6499e-11, 9.5690e-12, 5.7956e-12], device='cuda:0')" + }, + "61": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.8737e-13, -1.5834e-14, -2.1609e-14, -4.2316e-15, -2.8332e-15,\n -1.4918e-14, -7.1718e-14, -4.0738e-13, -9.1842e-14, 1.1175e-14,\n 3.0988e-14, 7.1549e-16, 5.8572e-15, -6.0898e-15, -1.2728e-13,\n -1.4164e-14, 5.8399e-15, -3.7523e-14, -1.5708e-13, -2.4353e-13,\n -1.5485e-15, -1.3308e-13, 6.1243e-15, -7.3714e-14, -5.9366e-15,\n -4.2070e-14, 6.4193e-14, 3.4444e-15, -2.9699e-14, -8.1007e-14,\n -1.8955e-13, -7.9405e-14, 6.4263e-16, -2.4894e-13, -3.4095e-15,\n 4.1514e-14, -1.1552e-15, 7.0272e-15, 4.8862e-14, 7.5686e-15,\n 1.6223e-14, -1.1982e-13, 2.4488e-15, -1.7462e-14, -1.6110e-14,\n -2.6156e-14, -3.6934e-13, 9.3105e-15, 5.7788e-14, -2.3630e-13,\n -2.0401e-13, -6.4432e-14, 2.2859e-15, -5.0085e-14, 1.8477e-14,\n 8.1606e-15, -4.4938e-15, 1.8582e-15, 7.2194e-14, -2.4484e-15,\n -7.6665e-14, 7.2075e-15, 7.9667e-14, -3.4712e-15, 4.3578e-14,\n 8.1609e-15, -1.1381e-14, -4.1012e-14, -1.0077e-13, 2.9698e-14,\n -3.2492e-13, -1.4968e-13, 2.3053e-14, -8.0458e-15, -3.3652e-13,\n 1.0021e-14, -1.7472e-13, -1.4621e-17, 1.0315e-13, -9.6043e-15,\n -1.8175e-13, -1.8076e-13, -1.6437e-13, -1.8322e-13, -9.1469e-14,\n 5.0501e-14, 3.6219e-14, -6.3961e-15, 3.9632e-15, 6.4662e-14,\n -3.1764e-14, -2.0842e-13, -8.8932e-14, -1.0386e-13, 2.5218e-14,\n -1.8103e-15, -1.6477e-13, -3.1675e-13, 1.8069e-14, 4.1032e-14,\n 1.2281e-14, -8.6877e-14, 8.7489e-14, -8.5813e-15, -2.3567e-14,\n -1.4524e-13, -8.3285e-15, -1.4254e-13, -1.6917e-14, 1.7575e-15,\n -1.0266e-15, 2.0310e-14, -5.8354e-17, 8.4227e-14, -2.5849e-13,\n 6.0218e-14, -2.2537e-13, -2.3903e-14, 1.0569e-14, 1.6049e-14,\n -4.1903e-15, -1.0051e-14, -4.5350e-13, -4.5288e-17, -1.6743e-13,\n 1.4948e-14, -6.0009e-15, -1.0084e-13, -5.4262e-15, 2.1405e-14,\n -2.0164e-15, -7.5079e-15, -9.6209e-15, 8.0336e-15, -1.8869e-14,\n -4.0214e-14, -2.0309e-14, -3.8771e-13, -6.8907e-14, 1.7149e-15,\n -9.0736e-15, 7.8646e-15, 8.7911e-15, 6.2034e-15, -3.7482e-13,\n 1.1574e-14, -1.4572e-13, -5.9602e-15, -1.4527e-15, -1.7037e-14,\n -1.8035e-14, -3.5291e-15, -5.4870e-15, -7.6199e-16, -3.9070e-15,\n 3.2649e-15, -6.2436e-15, 8.7428e-15, -1.7309e-13, -1.4483e-13,\n 2.3492e-14, -1.4384e-15, -5.4919e-14, -4.6405e-15, 2.4955e-14,\n -6.2439e-16, -2.2980e-14, 3.2674e-14, -1.6332e-13, -1.3269e-14,\n -6.2907e-14, 7.4858e-14, 2.3744e-15, -1.2018e-13, -2.2667e-13,\n -2.7977e-15, -2.0964e-13, 2.3206e-14, 8.5693e-15, 8.8422e-15,\n -4.4732e-13, 3.2888e-15, -9.7930e-15, 1.0891e-14, -7.3587e-15,\n -1.5813e-14, -7.3859e-14, -1.4487e-13, -5.4414e-15, 3.7537e-15,\n -1.3331e-14, -2.3341e-15, 1.1745e-15, 4.8326e-14, -2.0781e-16,\n -1.5665e-13, -4.9482e-15, -1.6673e-13, -2.4542e-15, 4.5082e-14,\n 4.7128e-14, -1.4556e-13, -8.8067e-15, -2.4589e-13, -4.0756e-14,\n -6.7049e-14, 2.4128e-15, 1.7540e-14, 2.4679e-14, -4.3586e-14,\n -1.2463e-13, 1.0670e-14, -3.6512e-14, -2.5045e-14, 2.3974e-14,\n 5.3766e-14, -2.0334e-15, 5.7657e-15, 3.5670e-15, -3.6602e-14,\n -2.4369e-15, 4.5020e-15, -5.9072e-14, -1.7765e-15, -1.6966e-13,\n 2.6393e-15, -4.7475e-13, -2.4634e-13, -1.5890e-14, -2.2065e-15,\n 1.9936e-14, -1.4556e-14, -2.4424e-14, -6.5744e-14, 7.8423e-14,\n 1.6953e-14, 7.5031e-14, -9.5749e-14, 3.0516e-14, -1.0048e-13,\n 6.4029e-14, -2.8258e-14, -3.2115e-15, 2.4788e-15, -2.7293e-15,\n -1.9778e-15, -4.7082e-14, -1.5143e-14, -8.4798e-14, -2.4450e-15,\n -1.8637e-13, -7.3865e-14, -6.4564e-14, -1.5105e-14, 8.1593e-14,\n -6.3751e-15, -2.9909e-14, -1.2248e-13, -1.1200e-14, -2.7919e-13,\n 2.5356e-14, 2.8867e-14, 9.3516e-15, -2.4730e-14, -1.2853e-13,\n -1.5311e-13, -4.6052e-15, 8.4389e-15, -7.4571e-16, -3.8935e-15,\n -2.9509e-14, 5.2529e-14, -3.1986e-15, 2.9203e-15, -1.0545e-13,\n 8.6991e-15, -1.4250e-13, -1.8873e-14, -1.2530e-13, 9.3720e-15,\n -2.5842e-13, -1.4548e-13, 2.5875e-13, -2.0343e-13, -1.2614e-14,\n -5.3759e-14, -1.7906e-15, 8.0127e-14, -2.3393e-13, -4.0953e-14,\n -2.0831e-13, -1.1214e-13, -1.4162e-14, -2.3294e-14, -5.9494e-15,\n 9.2151e-14, 6.5253e-15, -1.5611e-13, -2.1209e-15, -2.1456e-13,\n -3.2250e-14, -1.3080e-13, -8.5353e-14, 7.8474e-14, -5.0589e-14,\n -1.0261e-13, 5.1900e-15, 2.2911e-15, -1.0131e-13, -2.6099e-14,\n 1.5590e-14, 2.8582e-15, -1.0598e-14, -6.7590e-14, 8.8622e-14,\n -5.1944e-14, 3.5524e-14, -2.3656e-13, 8.6096e-16, 1.9130e-14,\n -7.2739e-15, 1.5794e-15, 1.6859e-14, -1.8142e-15, -6.5563e-14,\n -2.4295e-13, 8.9087e-15, -7.0686e-14, -2.0488e-14, -1.6573e-13,\n -1.1259e-14, 1.1136e-14, -1.9087e-13, 5.4244e-15, 6.3233e-14,\n -7.5336e-14, -2.3606e-14, -2.9254e-14, -1.4712e-13, -1.6411e-13,\n -1.6986e-13, -1.1649e-13, -1.9605e-14, -6.4491e-14, 1.4089e-13,\n 3.5194e-14, -2.2788e-13, 1.8074e-14, 2.8315e-15, -1.5334e-13,\n -1.9310e-14, 5.7519e-15, 1.5771e-15, 3.1763e-14, 3.2516e-15,\n -2.8038e-14, -9.8188e-16, 7.1482e-15, 2.6282e-15, -6.1896e-14,\n 1.6635e-16, 8.0491e-14, -7.7017e-15, -2.3083e-14, 3.1429e-14,\n -6.3104e-15, 1.0121e-14, -4.9864e-14, -2.4439e-14, -2.1325e-15,\n -3.0385e-15, 2.2666e-15, 1.3288e-14, -6.1678e-13, -6.8861e-14,\n -1.1414e-14, -3.2089e-16, -9.3945e-15, 4.7773e-15, 4.4536e-14,\n -9.8595e-15, -1.1214e-14, 1.5519e-15, -1.9427e-13, -1.4436e-13,\n -2.1024e-14, -1.9581e-13, 1.4503e-15, -1.6659e-14, -1.9987e-13,\n 6.5142e-15, -1.4416e-13, 5.3798e-14, -4.8088e-14, -8.1946e-15,\n 2.4631e-14, -8.7423e-14, -2.3606e-14, -1.4608e-13, -3.5014e-14,\n -8.7222e-17, 1.5133e-14, -1.8423e-13, -5.5702e-15, -2.2299e-13,\n 4.7057e-16, -1.7806e-13, -1.7197e-13, -2.2058e-13, 7.8669e-15,\n -2.5454e-13, 2.2768e-14, 1.7496e-14, -5.3344e-14, 1.3732e-13,\n -1.5943e-14, 7.3569e-15, -1.6412e-13, -2.1035e-13, 2.0102e-14,\n -1.6328e-13, 1.9053e-14, 4.5173e-14, -2.6149e-14, 3.8717e-14,\n -3.2208e-14, -2.8360e-15, 7.9769e-15, -3.0354e-14, 1.7983e-14,\n -1.9489e-14, -4.0941e-14, -7.9225e-15, 3.0067e-15, 2.9816e-14,\n -9.5405e-14, 1.1555e-15, -4.3728e-13, 1.5509e-15, 1.0518e-15,\n -1.2159e-13, -3.3483e-15, 9.7244e-15, -9.4357e-15, -1.1919e-13,\n -3.2777e-14, 6.0283e-15, -5.6919e-14, -9.3670e-16, -1.1649e-13,\n 9.7270e-15, 5.3357e-14, -6.5086e-14, 2.8339e-14, -1.0905e-13,\n 2.0846e-15, -3.8527e-15, -7.9294e-15, 9.5000e-15, 1.0321e-14,\n 5.4274e-14, -8.4470e-16, 1.1547e-14, -1.1450e-13, 6.1702e-15,\n 1.8788e-15, -2.9936e-15, -2.2077e-14, -1.8759e-14, -1.2315e-13,\n -5.0038e-14, -1.0219e-13, 2.4320e-16, -1.0510e-14, -1.4886e-15,\n -1.3025e-14, -9.5726e-15, -3.9622e-14, 5.9165e-14, -2.4430e-14,\n 3.4266e-14, -2.8891e-14, 2.6824e-14, -3.6733e-15, -1.8803e-14,\n 6.7125e-15, -1.0766e-13, -2.7199e-14, -5.7941e-14, -2.3251e-13,\n -2.7147e-15, -2.7942e-13, -1.3977e-14, 5.6082e-14, 6.3717e-15,\n -6.2087e-14, -2.3492e-13, 1.2485e-13, 1.5593e-13, 8.0013e-15,\n 3.6277e-14, -2.7042e-13, 5.8912e-15, 3.9834e-14, -3.2523e-15,\n -2.2585e-13, -6.8752e-14, -2.1695e-13, -1.1424e-14, -9.0334e-16,\n 1.6206e-13, 2.9439e-15, -9.4451e-15, -4.9857e-13, 8.5410e-14,\n -7.0457e-15, 4.1015e-14, -2.1501e-14, 1.1463e-14, 1.4573e-15,\n 2.9712e-14, 1.1710e-15, 7.0198e-15, -5.1629e-15, 9.3238e-15,\n -8.0508e-15, -1.0514e-15, -4.0666e-13, -2.2797e-14, 4.0432e-15,\n 9.6041e-16, -1.2304e-14, -1.7990e-14, 3.0337e-15, -5.3686e-14,\n 2.3982e-14, -2.0953e-16, -7.0683e-14, -7.6996e-14, -2.0263e-13,\n -3.0960e-13, -2.5868e-14, -2.3591e-13, -6.0513e-15, -1.1181e-13,\n -2.7050e-13, -2.5345e-14, 8.1752e-14, 2.9595e-14, -1.6844e-14,\n -5.8264e-14, 3.6229e-15, -1.0660e-13, 4.7865e-14, 6.1070e-16,\n -2.1609e-13, 1.5103e-14, -4.8045e-15, 6.6795e-14, -2.9553e-14,\n 1.2431e-14, -3.0725e-13, 1.7236e-15, -4.1005e-13, 2.6474e-15,\n 4.2897e-14, 2.6847e-14, -9.0292e-14, -1.3827e-13, 3.7954e-15,\n 1.8985e-14, -2.6900e-13, 3.6164e-14, 2.1722e-14, -1.6360e-14,\n 9.7368e-14, -1.1408e-15, -7.1688e-14, -4.0359e-15, 3.1456e-14,\n 2.9259e-14, -1.0286e-13, 8.5451e-15, 1.0400e-13, -8.8672e-16,\n 8.5837e-16, 5.1999e-17, 7.5469e-14, -4.2529e-14, 1.0376e-15,\n 1.9250e-14, -2.0460e-14, -2.9731e-13, -1.1658e-14, 7.4450e-15,\n 8.2576e-14, -5.4512e-14, -5.0660e-15, 1.6654e-14, -3.2580e-14,\n -1.2505e-13, 6.1296e-14, 4.2354e-16, -1.6636e-13, -1.5594e-14,\n 9.6150e-15, -2.7965e-13, -2.2064e-13, -2.2431e-13, -1.1257e-13,\n -1.9986e-15, -5.7807e-15, -1.3391e-16, -1.1863e-14, 1.4814e-14,\n -4.2331e-14, -1.2163e-13, 5.4442e-14, 4.9310e-16, 4.7722e-14,\n -2.3517e-14, 5.2215e-14, -1.6295e-14, 1.4445e-14, -1.6885e-14,\n -2.9490e-14, 1.8456e-14, -8.0839e-14, -2.0636e-16, -2.4588e-13,\n -3.8525e-15, 9.1017e-15, 3.8436e-14, -4.4175e-16, -3.2767e-15,\n -7.5290e-15, 1.8832e-14, 8.5023e-15, 1.3384e-14, -6.8138e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([8.2845e-10, 5.7379e-11, 1.5448e-09, 4.6257e-12, 5.9356e-11, 1.5069e-11,\n 3.6985e-11, 3.9836e-09, 1.4589e-09, 2.6789e-10, 7.9000e-12, 7.7755e-10,\n 2.2120e-11, 3.5141e-12, 5.0851e-11, 1.3458e-12, 1.2588e-11, 5.3019e-09,\n 2.5224e-09, 2.0953e-10, 1.6244e-10, 2.9190e-10, 2.2533e-10, 5.7342e-10,\n 6.6351e-10, 6.0010e-11, 1.1690e-11, 8.9297e-12, 4.6860e-11, 7.3761e-11,\n 4.4526e-09, 3.1189e-10, 1.0686e-09, 2.2515e-09, 1.4077e-10, 2.7147e-12,\n 3.7849e-11, 2.1624e-10, 4.0895e-12, 7.2334e-11, 2.3520e-12, 1.2125e-09,\n 2.0056e-10, 5.9664e-12, 1.3010e-10, 8.9287e-13, 1.1523e-09, 1.5918e-11,\n 1.4689e-11, 1.4560e-09, 2.7783e-10, 1.4804e-09, 7.9855e-12, 1.6504e-10,\n 2.9225e-10, 5.0741e-10, 4.6264e-11, 1.3390e-11, 1.3512e-11, 3.1299e-10,\n 1.7166e-10, 1.6128e-10, 1.5169e-11, 1.9503e-12, 6.9151e-10, 2.5422e-10,\n 6.8187e-12, 2.3370e-09, 2.1415e-10, 4.5911e-09, 5.6159e-10, 1.5556e-09,\n 2.1555e-10, 2.8927e-10, 1.7520e-10, 8.5685e-13, 1.0412e-09, 7.2850e-10,\n 4.4800e-09, 8.3368e-10, 1.4329e-10, 1.1412e-11, 1.3154e-09, 2.4584e-09,\n 1.2774e-09, 9.9303e-11, 1.3948e-11, 3.0627e-10, 1.8624e-11, 2.0647e-12,\n 1.1095e-09, 2.8683e-11, 3.1402e-10, 4.4552e-10, 1.6538e-10, 1.8807e-09,\n 1.6087e-10, 1.9427e-09, 3.9248e-11, 5.8608e-10, 3.7901e-11, 8.7737e-13,\n 1.0473e-11, 3.5135e-11, 8.5507e-12, 1.0602e-09, 5.7385e-10, 9.7478e-10,\n 1.5039e-12, 2.0740e-12, 1.3227e-09, 2.6187e-10, 2.0946e-09, 1.5136e-11,\n 3.5943e-10, 4.1075e-12, 6.9081e-11, 6.8549e-09, 5.5952e-10, 1.7423e-10,\n 2.2713e-11, 9.1877e-11, 4.1640e-10, 9.7568e-11, 9.4747e-10, 5.4313e-12,\n 2.6765e-11, 9.8471e-10, 2.1661e-11, 5.8115e-12, 5.4453e-11, 9.1638e-11,\n 1.6530e-09, 8.5639e-11, 6.3042e-10, 3.1591e-09, 4.8477e-12, 4.3456e-10,\n 1.4834e-09, 3.8552e-12, 2.4302e-10, 1.0059e-09, 3.9969e-12, 9.4987e-11,\n 7.3688e-11, 2.9691e-12, 1.5604e-10, 6.5223e-10, 1.4885e-10, 1.6412e-09,\n 5.6427e-12, 2.8326e-11, 1.0431e-09, 2.4526e-10, 4.6077e-11, 3.8795e-11,\n 1.2702e-11, 2.4931e-11, 3.5540e-10, 4.3049e-10, 2.3159e-09, 2.8767e-11,\n 5.2085e-09, 1.0217e-10, 1.9731e-11, 1.8715e-09, 1.2089e-10, 5.0091e-11,\n 2.2318e-09, 6.3653e-10, 1.1064e-09, 2.9235e-11, 1.8842e-11, 8.0746e-10,\n 2.0988e-10, 3.6535e-09, 9.5013e-11, 1.5239e-09, 1.1062e-12, 1.0647e-11,\n 5.2858e-10, 3.6796e-10, 1.1397e-10, 1.9625e-10, 2.0194e-12, 2.9667e-10,\n 2.5316e-09, 3.1628e-09, 2.8966e-11, 1.4311e-09, 1.0356e-09, 1.5920e-11,\n 2.2123e-11, 5.1314e-12, 2.6174e-09, 4.3256e-10, 7.5606e-12, 3.4916e-10,\n 1.2436e-09, 1.1617e-10, 8.7885e-12, 4.9829e-12, 1.6663e-10, 2.2919e-09,\n 8.9905e-11, 9.2474e-10, 1.6618e-11, 8.5087e-12, 5.2823e-11, 4.3081e-10,\n 4.3084e-10, 3.5812e-12, 6.3914e-10, 7.5105e-10, 8.6235e-11, 1.4134e-10,\n 1.8631e-10, 2.5821e-10, 1.6572e-10, 6.1045e-09, 2.0584e-10, 1.2710e-11,\n 5.2839e-10, 6.1196e-13, 4.7091e-12, 5.0990e-12, 3.4098e-09, 6.4586e-10,\n 1.3309e-11, 3.1460e-11, 5.7167e-12, 1.3037e-12, 2.5134e-09, 7.2438e-11,\n 4.6245e-11, 1.2440e-11, 8.8197e-13, 4.3386e-09, 4.3271e-11, 2.4624e-10,\n 2.9735e-11, 1.3892e-12, 3.0332e-10, 1.2304e-11, 2.5307e-12, 2.3961e-09,\n 9.4727e-10, 1.5128e-09, 4.0422e-10, 9.9062e-11, 2.3353e-09, 3.4364e-10,\n 9.2713e-10, 3.4759e-11, 1.4310e-11, 1.7520e-09, 1.0439e-09, 7.0163e-11,\n 4.4183e-10, 8.0040e-10, 1.2827e-11, 2.2656e-12, 2.9826e-11, 5.9652e-10,\n 2.5975e-09, 7.9075e-11, 1.5317e-10, 3.3953e-11, 6.6339e-10, 3.8620e-11,\n 7.9822e-12, 8.8179e-11, 2.0586e-10, 1.9994e-10, 6.9950e-10, 5.2958e-12,\n 6.8730e-11, 4.3551e-11, 2.6834e-09, 1.1370e-11, 3.3767e-10, 2.4569e-09,\n 7.0198e-11, 2.4815e-09, 6.4669e-12, 4.6701e-10, 4.4181e-11, 3.8472e-12,\n 6.0251e-09, 1.5231e-11, 3.0309e-09, 7.6819e-10, 5.8294e-11, 6.8602e-11,\n 7.7859e-12, 1.0673e-10, 1.6412e-11, 2.8389e-10, 1.3773e-10, 3.1024e-10,\n 7.4135e-10, 3.1580e-10, 9.3962e-11, 1.5132e-11, 8.3478e-11, 1.1270e-09,\n 3.2401e-11, 4.4164e-11, 3.1551e-10, 1.6002e-12, 3.3249e-10, 1.7858e-11,\n 6.7761e-10, 4.0631e-10, 5.0104e-12, 1.5396e-10, 1.0838e-11, 1.0212e-09,\n 3.4525e-12, 1.6993e-09, 2.1337e-10, 1.5255e-11, 2.6802e-12, 7.4561e-12,\n 1.1334e-10, 1.6276e-10, 9.2034e-11, 2.3483e-10, 5.1547e-11, 1.3894e-09,\n 2.9182e-11, 1.1961e-12, 1.5302e-09, 7.2301e-12, 7.7062e-12, 7.8380e-10,\n 1.5484e-09, 1.3887e-10, 2.6312e-10, 8.2305e-13, 9.7534e-12, 3.6451e-10,\n 3.7842e-12, 5.3748e-10, 2.7318e-11, 4.8207e-11, 1.0806e-09, 3.0591e-09,\n 8.7142e-12, 3.4200e-09, 4.4081e-12, 3.1838e-11, 3.6993e-10, 4.5263e-11,\n 5.0882e-12, 4.7911e-11, 2.4673e-10, 5.7749e-12, 3.3959e-12, 4.2875e-10,\n 2.4260e-10, 6.0470e-11, 2.6296e-11, 2.4740e-11, 6.3129e-11, 9.9019e-12,\n 3.4441e-12, 1.2885e-09, 1.6533e-10, 2.4251e-09, 2.8543e-09, 3.9526e-10,\n 5.4753e-11, 7.6453e-10, 4.3655e-10, 1.5671e-11, 4.5582e-10, 1.6666e-11,\n 2.5418e-12, 2.0705e-11, 4.9306e-11, 1.7319e-09, 5.4305e-10, 3.8728e-11,\n 3.3987e-10, 3.4121e-09, 8.3624e-10, 2.4927e-11, 4.2793e-11, 1.7925e-09,\n 9.3046e-10, 5.3540e-10, 3.5913e-11, 2.4518e-09, 6.3729e-12, 7.7177e-12,\n 2.0170e-09, 7.0823e-11, 1.7133e-09, 1.7958e-09, 8.3011e-12, 1.2569e-11,\n 1.4527e-10, 9.7626e-10, 1.4165e-09, 1.2275e-10, 2.9605e-10, 1.5350e-10,\n 4.7627e-10, 3.9111e-10, 1.1419e-10, 7.3086e-12, 3.1623e-11, 1.3625e-10,\n 1.0678e-10, 5.0584e-10, 7.9495e-11, 3.1672e-09, 9.7593e-11, 9.0940e-10,\n 3.9255e-10, 2.4130e-10, 3.3761e-11, 9.1213e-11, 7.1182e-10, 5.0813e-10,\n 4.1481e-10, 3.9907e-12, 1.0768e-10, 1.0797e-11, 4.3556e-10, 1.0610e-09,\n 1.1367e-10, 1.2788e-10, 1.4218e-10, 1.6419e-10, 1.0944e-09, 3.5957e-10,\n 1.3285e-12, 3.1588e-12, 9.3170e-11, 4.3041e-11, 1.4385e-10, 5.4823e-10,\n 1.1114e-09, 2.0233e-10, 5.0749e-10, 2.9267e-10, 2.1108e-11, 1.2292e-10,\n 5.6203e-11, 2.6345e-12, 8.8259e-11, 3.2050e-10, 2.0499e-09, 1.0123e-09,\n 5.0756e-11, 4.4789e-11, 2.2937e-10, 1.7569e-10, 2.6612e-11, 1.0080e-09,\n 3.9572e-12, 2.3508e-10, 1.9265e-10, 2.1318e-09, 5.6003e-10, 1.6230e-10,\n 4.6843e-12, 6.3974e-11, 1.1084e-09, 6.6291e-09, 2.1745e-10, 5.1308e-12,\n 2.4204e-09, 9.6406e-10, 3.9013e-12, 6.4631e-12, 1.5375e-11, 4.3150e-12,\n 9.4583e-12, 1.3867e-11, 3.6475e-09, 2.9432e-12, 4.8306e-11, 1.9000e-10,\n 6.4717e-10, 1.3997e-12, 6.8732e-11, 2.3979e-10, 9.7345e-12, 8.3902e-10,\n 5.3036e-12, 3.7898e-12, 3.6351e-11, 1.6280e-09, 5.1842e-11, 3.1155e-12,\n 5.1990e-10, 5.6093e-10, 3.1252e-11, 9.2134e-10, 1.7034e-10, 3.4598e-11,\n 2.4848e-10, 2.6722e-11, 3.3984e-10, 4.3177e-10, 2.8763e-12, 8.5656e-11,\n 5.8091e-12, 2.1335e-11, 1.2493e-09, 1.2908e-09, 7.6915e-10, 2.7074e-10,\n 5.1292e-12, 1.2444e-10, 4.1920e-09, 3.4694e-12, 1.0120e-09, 2.9271e-11,\n 5.3930e-12, 6.1311e-10, 1.1341e-11, 5.4426e-11, 2.5223e-12, 1.0448e-09,\n 4.7536e-11, 2.0655e-12, 7.8599e-11, 4.6309e-09, 2.1310e-09, 2.3842e-10,\n 6.2316e-12, 2.4196e-10, 6.1733e-11, 8.7261e-09, 4.3032e-10, 6.2293e-11,\n 2.4937e-10, 6.2013e-10, 1.3381e-09, 3.2583e-12, 2.5042e-10, 1.7675e-10,\n 6.1277e-10, 2.3929e-10, 1.4300e-11, 1.4549e-09, 1.9600e-09, 8.6926e-11,\n 5.2135e-10, 1.4702e-12, 7.7224e-09, 3.9583e-09, 3.3165e-10, 6.8785e-10,\n 2.8157e-12, 1.1187e-09, 2.2572e-10, 1.7588e-09, 2.0572e-11, 9.2832e-10,\n 9.5995e-10, 5.5793e-12, 1.7214e-11, 1.5592e-09, 2.5580e-09, 3.0221e-10,\n 7.4063e-11, 1.0969e-09, 1.6092e-09, 2.7051e-10, 7.6350e-10, 2.0209e-11,\n 6.1227e-10, 1.9565e-11, 1.2647e-10, 2.6936e-11, 5.8411e-11, 1.0082e-11,\n 9.5692e-11, 5.6128e-11, 1.3218e-11, 6.9035e-11, 3.2619e-12, 9.2039e-12,\n 1.9136e-09, 1.6496e-10, 1.9801e-11, 3.8026e-11, 1.3868e-09, 3.9592e-11,\n 5.7048e-12, 1.5126e-12, 1.4066e-10, 8.5147e-10, 1.9350e-12, 6.4272e-11,\n 5.6690e-11, 1.7494e-11, 2.6048e-11, 3.4732e-10, 1.0876e-09, 1.8771e-11,\n 1.3000e-09, 8.1946e-10, 5.7121e-10, 3.3903e-10, 5.7391e-10, 2.2386e-11,\n 5.1278e-12, 8.3433e-10, 5.0900e-10, 4.0960e-11, 1.2854e-11, 2.8215e-11,\n 2.0301e-10, 3.2492e-11, 2.9871e-10, 5.0195e-10, 4.4699e-12, 4.9530e-12,\n 4.3219e-09, 7.8952e-10, 1.8244e-09, 3.3414e-09, 4.2443e-10, 1.6595e-09,\n 4.2322e-09, 2.4945e-11, 1.5441e-11, 5.9771e-12, 1.1175e-10, 4.7958e-11,\n 6.1442e-12, 7.0334e-12, 2.7510e-12, 7.3207e-11], device='cuda:0')" + }, + "62": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.4522e-15, -1.6151e-13, -3.1862e-15, ..., -6.8849e-14,\n -1.5314e-13, -5.9563e-14],\n [ 3.9192e-15, -1.0797e-13, -2.0552e-15, ..., -4.5898e-14,\n -1.0175e-13, -4.0786e-14],\n [-6.2892e-15, 1.1139e-13, -1.5195e-15, ..., 4.7501e-14,\n 7.7985e-14, 4.2363e-14],\n ...,\n [ 3.6776e-15, -2.2476e-14, 4.1167e-15, ..., -1.0802e-14,\n -1.0237e-13, -2.8624e-14],\n [-4.6605e-15, 2.7986e-14, -5.1821e-15, ..., 1.3372e-14,\n 1.2999e-13, 3.5717e-14],\n [-9.3007e-15, 5.7862e-14, -1.0280e-14, ..., 2.8185e-14,\n 2.6272e-13, 7.2260e-14]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.6612e-12, 1.1786e-11, 4.3793e-11, ..., 8.3007e-12, 1.4355e-12,\n 2.5982e-12],\n [8.9137e-13, 1.5882e-12, 1.4297e-11, ..., 4.4816e-12, 4.8386e-13,\n 5.6088e-13],\n [3.4024e-13, 1.0182e-12, 1.3374e-11, ..., 4.9635e-13, 2.8952e-13,\n 2.2181e-13],\n ...,\n [6.6106e-11, 2.7278e-10, 1.0089e-09, ..., 4.5271e-11, 4.5880e-10,\n 1.7960e-10],\n [1.5873e-11, 1.2319e-10, 1.1871e-10, ..., 1.4283e-11, 7.2006e-10,\n 1.7445e-10],\n [6.9631e-11, 2.2974e-10, 9.8506e-10, ..., 2.6428e-11, 3.5004e-10,\n 2.1423e-10]], device='cuda:0')" + }, + "63": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-8.3563e-14, -5.8277e-14, 5.4188e-14, ..., -2.7995e-14,\n 3.5064e-14, 7.1307e-14], device='cuda:0')", + "exp_avg_sq": "tensor([1.1001e-10, 1.3577e-11, 2.9527e-11, ..., 4.5372e-09, 4.4388e-10,\n 2.9252e-09], device='cuda:0')" + }, + "64": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.9516e-14, -9.2091e-16, 3.7832e-14, ..., -3.7075e-14,\n 7.3371e-14, -3.1870e-14],\n [-7.4822e-14, -4.7501e-15, 6.2978e-14, ..., -5.2481e-14,\n 1.0544e-13, -4.4483e-14],\n [-2.8972e-14, -3.2555e-15, 1.5928e-14, ..., -4.9461e-14,\n 9.7987e-14, -4.1764e-14],\n ...,\n [ 9.0802e-14, 6.6637e-15, -7.3903e-14, ..., 7.2292e-14,\n -1.4408e-13, 6.3529e-14],\n [ 5.8544e-14, 3.4782e-15, -5.0761e-14, ..., 3.8743e-14,\n -7.7517e-14, 3.4673e-14],\n [ 1.2281e-13, 7.7835e-15, -1.0188e-13, ..., 9.6601e-14,\n -1.9350e-13, 8.3051e-14]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.3925e-10, 1.4614e-09, 2.5356e-10, ..., 1.1834e-08, 1.5587e-09,\n 3.4020e-09],\n [2.5221e-10, 2.2903e-10, 1.2059e-10, ..., 2.4473e-10, 1.0214e-10,\n 2.1929e-10],\n [1.5232e-10, 1.5327e-10, 1.0343e-10, ..., 3.8186e-10, 1.0802e-10,\n 1.8650e-10],\n ...,\n [4.2526e-10, 1.1732e-10, 2.9464e-10, ..., 9.6558e-11, 3.0161e-10,\n 4.9156e-10],\n [2.1883e-10, 1.7893e-10, 1.6643e-10, ..., 2.6514e-10, 1.4101e-10,\n 1.7188e-10],\n [3.9904e-10, 7.1472e-10, 4.6117e-10, ..., 5.7232e-09, 8.4251e-10,\n 1.8161e-09]], device='cuda:0')" + }, + "65": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-2.7455e-14, -5.1092e-14, -2.9462e-14, 2.5469e-14, -4.1949e-14,\n 7.4347e-14, -5.3855e-15, -6.8494e-15, -7.8382e-14, -4.7937e-15,\n -2.0658e-14, 8.8626e-15, -2.5925e-14, 5.0327e-14, 8.0431e-14,\n -6.9298e-14, -5.5848e-14, -4.8763e-15, -5.1305e-15, 5.7528e-14,\n -2.4718e-14, -5.8254e-14, -2.9290e-14, -2.3277e-14, 2.4438e-14,\n -2.7497e-15, 2.6529e-14, -1.1581e-14, 1.9259e-14, -5.2249e-14,\n 1.3885e-14, 1.8494e-14, -2.8591e-14, -7.3796e-15, -1.4125e-14,\n 3.6205e-14, -3.9821e-14, 6.0703e-14, 2.3161e-14, 2.7002e-14,\n -9.6590e-14, 2.3371e-14, -1.9983e-14, -2.6537e-14, 1.0538e-13,\n -3.8018e-14, 2.3796e-14, 8.1824e-14, 7.2333e-14, -5.7871e-14,\n 1.5174e-14, 2.0119e-14, 9.4018e-14, 4.7496e-14, -5.8496e-14,\n 3.9450e-14, 5.3427e-14, 4.1291e-15, -1.3819e-14, -6.6774e-14,\n 6.1057e-15, -1.0938e-14, 2.8684e-14, 7.5582e-14, -8.3594e-14,\n -4.2902e-14, 3.4224e-14, -1.4982e-14, -3.9088e-14, 5.9377e-14,\n -5.4604e-14, 5.8604e-14, -3.8393e-14, -7.6616e-14, -1.6090e-14,\n -6.7895e-14, -1.5388e-14, 5.4766e-14, 2.3478e-15, 1.8769e-14,\n 2.3101e-15, -1.2332e-14, -5.2357e-15, 1.1532e-14, -3.9810e-14,\n 8.6557e-15, 8.0456e-14, 5.2682e-14, -1.2103e-14, -3.2784e-14,\n -2.5223e-14, 4.9812e-14, 1.0896e-14, -6.2876e-14, 1.3487e-15,\n 6.1014e-14, 5.4915e-14, -2.3661e-14, 7.8791e-15, 1.8180e-14,\n -6.3955e-14, 3.2612e-14, 3.4580e-14, 4.6723e-15, -3.3007e-14,\n -3.0941e-14, 9.7590e-15, -8.7158e-15, -4.3708e-14, -1.4709e-14,\n 1.7683e-15, 6.5908e-15, -1.5190e-14, -3.0605e-14, -5.7202e-14,\n -3.9833e-14, 4.8830e-14, 5.3862e-15, -5.4807e-14, -5.2994e-14,\n 6.8478e-14, 7.9404e-15, -9.3688e-14, -5.7162e-14, -6.0413e-14,\n 3.7641e-14, 5.6386e-14, 4.2864e-14, -6.4355e-14, -2.7440e-14,\n -6.3356e-14, 5.7211e-14, -9.6404e-15, -3.8487e-14, -6.6961e-14,\n 4.7162e-14, 5.0931e-14, -1.0254e-14, -1.6297e-14, 7.0827e-14,\n -6.6291e-14, -6.6486e-14, 9.6985e-14, 1.6046e-14, 8.4602e-14,\n -1.1160e-14, 5.2396e-14, 1.0625e-15, 8.2614e-14, -4.0462e-14,\n 2.1613e-14, 1.2315e-14, 4.4526e-14, -4.1660e-14, 1.2508e-14,\n 7.5103e-14, 4.1005e-14, -5.3516e-14, -2.0581e-14, -1.2572e-14,\n 1.3977e-14, -9.3297e-14, 6.4663e-14, 1.4902e-14, -6.5633e-14,\n -4.0031e-14, -2.4084e-14, 2.5921e-14, -4.0240e-15, 1.1567e-14,\n -6.8764e-14, 6.4542e-14, 6.3165e-14, -1.1628e-13, 8.0917e-14,\n 9.9799e-14, 7.3175e-14, 4.5125e-14, -3.9049e-14, 9.0092e-14,\n -6.1911e-14, -1.0241e-13, 5.5528e-14, 9.0847e-14, 5.3498e-14,\n 4.7680e-14, 2.5973e-14, -3.5847e-14, 4.7110e-14, -6.5484e-14,\n -6.8263e-14, 2.0629e-14, 1.0822e-14, 9.3096e-16, 5.2795e-14,\n 4.5973e-14, 8.8266e-14, -9.1307e-14, 4.9450e-14, -7.7743e-14,\n 5.6038e-14, 4.7137e-14, -7.9094e-14, 1.3859e-14, 7.1830e-14,\n -5.7149e-14, -3.5231e-14, -5.9262e-14, 4.7614e-14, 6.3183e-14,\n -3.4844e-14, -6.2153e-14, -3.4399e-14, 7.0504e-14, -3.2667e-14,\n 6.9217e-14, 8.2840e-14, 7.9613e-14, 4.4454e-14, 3.5532e-14,\n -7.1209e-14, -7.5852e-14, -3.6467e-14, 3.6183e-14, 2.1880e-14,\n 5.4608e-14, -5.6685e-14, 3.9258e-14, -6.6956e-14, -2.4302e-14,\n -1.0310e-13, -1.4271e-14, 5.5230e-14, -6.1376e-14, 6.9129e-14,\n 2.4663e-14, -1.0221e-13, 1.7337e-14, -4.6265e-14, -3.5084e-14,\n 4.3485e-14, -8.1283e-14, -5.7263e-14, 1.6393e-14, 6.6724e-14,\n -8.8340e-14, -4.8625e-14, -9.0753e-16, 3.4025e-15, 7.9283e-14,\n -3.1933e-15, -1.8320e-14, -7.5398e-15, 9.1445e-14, 2.1047e-14,\n -4.1922e-14, -6.1566e-14, -1.5520e-14, 3.9506e-14, -7.5177e-14,\n -2.1590e-14, -4.1978e-15, -4.0807e-14, -4.9954e-14, 8.6403e-14,\n -2.9234e-14, 3.8339e-14, -3.8316e-14, -5.4997e-14, 6.8130e-14,\n 7.5984e-14, 4.6482e-15, 2.6825e-14, 6.9900e-14, 1.6991e-14,\n 1.4996e-14, 9.3472e-14, -7.3049e-14, 3.3113e-14, 5.8710e-14,\n -1.8466e-14, 3.7401e-15, 6.0137e-15, 2.6409e-14, -3.8609e-14,\n -4.6380e-14, 3.7836e-14, -4.0291e-14, 4.7013e-14, 5.7245e-14,\n -1.6882e-14, 5.2983e-14, -1.7420e-14, -5.1258e-14, 4.1936e-14,\n 8.1775e-14, -2.1788e-14, 9.2415e-15, 1.1342e-14, 7.2423e-14,\n -4.7581e-14, 1.2030e-14, -3.2378e-14, -8.8253e-14, -6.6391e-14,\n 5.3602e-14, -6.7535e-14, -6.7646e-14, 1.5032e-14, -8.9386e-14,\n 2.8857e-14, -6.6746e-14, -6.1674e-15, 1.4097e-15, 2.8455e-14,\n 4.4267e-14, -5.3816e-14, 6.3759e-14, 2.8202e-14, -3.5822e-14,\n -2.9140e-14, 6.7244e-16, 1.6093e-14, -4.2076e-15, 4.8392e-14,\n -6.8344e-14, -9.8635e-14, -9.1228e-14, 1.5427e-14, 8.0351e-14,\n 2.7273e-14, 8.3751e-14, 1.9578e-14, 6.1265e-14, -9.6694e-14,\n -7.9509e-14, -2.7429e-14, -1.5179e-14, -4.0859e-14, 2.2538e-14,\n -3.6366e-14, 2.9259e-14, -5.7252e-14, -2.1191e-14, -2.7172e-14,\n -4.6003e-15, 5.3488e-14, -6.2896e-14, 5.4787e-14, 2.4052e-15,\n 1.2796e-14, 2.3683e-16, -1.2904e-14, -5.9306e-14, 7.9468e-14,\n 7.8552e-14, -2.0171e-14, 3.9740e-14, -4.1129e-14, 1.6118e-14,\n 5.4524e-14, -2.0224e-14, -3.3325e-14, 4.5848e-14, -5.7499e-14,\n 7.1712e-14, -2.9988e-14, -1.1676e-13, 4.7798e-14, 6.9082e-14,\n -4.1215e-14, -6.3183e-14, 6.4612e-14, 1.7003e-14, 7.3907e-15,\n -7.3365e-15, -1.3129e-14, -4.9083e-16, 8.6155e-14, 3.2978e-14,\n 1.7897e-14, 9.3719e-14, -2.5627e-14, 1.7001e-14, 2.0598e-14,\n -6.2285e-15, 3.7681e-14, 6.3103e-14, 3.9370e-14, 3.1440e-14,\n 6.7583e-14, 1.4421e-14, 2.7782e-14, 4.1753e-14, -6.9122e-14,\n 3.1449e-14, -6.2547e-14, -7.7646e-15, -1.8476e-14, 1.0471e-13,\n 3.1968e-14, 6.4681e-15, -7.1443e-14, 2.2035e-14, 4.2739e-15,\n 8.1007e-14, -6.2097e-14, 3.0176e-14, -5.8773e-14, -2.6788e-15,\n -4.6039e-14, -1.2212e-14, 5.3785e-14, 2.0200e-14, -3.6192e-14,\n -8.1490e-14, 6.1550e-14, 7.0371e-14, -2.7722e-14, 1.9176e-16,\n -1.4290e-14, -3.6708e-14, -6.7529e-14, 2.5952e-14, 1.3091e-14,\n 3.2202e-14, 3.9381e-14, 7.2105e-14, 4.5107e-14, -1.9580e-15,\n 2.1775e-14, 2.2901e-14, 1.6621e-14, -1.4527e-14, -3.2015e-14,\n 6.1543e-14, -6.9082e-15, 1.5997e-14, -3.5051e-14, -2.9971e-14,\n -4.6821e-14, 1.9267e-14, -1.3677e-14, 3.0341e-14, -5.6516e-14,\n -1.7697e-14, -2.6869e-14, -8.8483e-14, -2.9207e-14, -4.5065e-14,\n 5.3397e-14, 8.6310e-14, -3.3179e-14, -4.4607e-14, 1.9611e-14,\n -7.0840e-14, -5.4001e-16, 8.2367e-14, -1.2039e-14, -1.3897e-14,\n -2.8404e-14, -5.0855e-14, -2.5729e-14, 2.6400e-14, 4.8515e-14,\n -2.2269e-15, 3.6790e-14, 3.4604e-14, -1.8417e-14, 2.9404e-14,\n 2.1449e-14, 6.9825e-14, 4.3332e-14, -1.2495e-14, 1.3813e-14,\n -9.0258e-15, 5.1171e-14, 5.4685e-14, -1.0537e-14, -3.2489e-14,\n -1.0666e-13, 4.9391e-14, -6.5537e-15, -1.8911e-14, -1.3864e-14,\n 2.8211e-15, 1.8050e-14, -1.7129e-14, -4.7091e-14, -3.3801e-14,\n 3.3413e-14, -4.2308e-14, 1.0135e-13, -5.2911e-14, -1.4877e-14,\n 4.2800e-14, -5.3687e-14, -3.6765e-14, 1.8979e-14, 4.5554e-14,\n 6.6384e-14, -1.0183e-13, -1.8175e-15, 3.9854e-16, -5.9341e-14,\n 3.0739e-14, 5.2019e-14, -5.3027e-14, -3.9999e-14, 2.3133e-14,\n 2.5982e-14, 3.2093e-15, -7.1506e-14, 6.0089e-14, -1.0831e-13,\n -2.9694e-14, -2.4885e-14, 2.7938e-14, 8.7003e-14, -1.7211e-14,\n -7.7418e-14, -6.1439e-14, 2.9698e-14, -1.2342e-13, -5.3717e-14,\n 1.0576e-14, -9.9168e-14, -3.5576e-14, -3.9742e-14, -2.6120e-14,\n 7.3219e-14, 1.1241e-14, 3.2313e-14, -2.5977e-14, -1.0274e-14,\n 4.3770e-14, 5.8636e-14, -3.9440e-14, 2.3123e-14, -6.4245e-14,\n -1.8354e-14, -7.2898e-14, 7.3601e-14, -2.3964e-14, 1.0957e-13,\n -1.7687e-16, 4.4755e-14, 3.1294e-14, -5.4218e-15, 1.3203e-14,\n 7.6138e-14, 4.0658e-14, -7.7253e-14, 6.9908e-14, 1.3503e-14,\n -6.3843e-14, 4.1145e-14, 3.8452e-14, 3.9704e-14, -2.8982e-14,\n 5.7165e-14, -1.6521e-15, -8.5004e-14, -1.1384e-16, -4.8660e-14,\n 2.0167e-15, 1.4606e-14, 3.8522e-14, 7.8739e-14, 6.2236e-14,\n 6.5286e-14, -2.6552e-14, 6.4402e-14, 3.3275e-14, 2.4280e-15,\n -7.8080e-14, -3.8608e-14, -1.8314e-14, 2.2148e-14, 3.9762e-14,\n 4.7163e-14, 3.1021e-14, 2.5916e-14, -2.6778e-14, 8.7576e-14,\n 2.3008e-14, 5.4382e-14, 8.0695e-14, 8.5895e-14, 3.3928e-14,\n -2.4122e-14, -6.6259e-14, 2.0796e-14, -8.1732e-15, 1.7103e-14,\n -4.1412e-14, -3.0778e-16, -2.5460e-15, -8.7937e-15, 7.9673e-14,\n -6.5956e-14, -4.5943e-14, -5.8178e-14, 3.1967e-15, -1.2415e-14,\n -7.0377e-14, -2.8688e-14, -4.8194e-14, 4.2468e-14, -6.5748e-14,\n 3.0821e-14, 4.3108e-14, -5.9269e-15, -1.4843e-14, -1.2932e-14,\n -2.5965e-14, -1.6651e-14, -2.5456e-14, -1.4458e-14, 1.6524e-14,\n 5.0206e-15, 4.8520e-14, 1.6036e-14, 2.5304e-14, -1.4232e-14,\n -2.4246e-14, -6.2607e-14, -3.4913e-14, 9.1721e-15, -8.1815e-14,\n -4.7169e-14, -3.5095e-14, 3.4068e-14, -5.1047e-14, 9.3374e-14,\n 2.3146e-14, 7.3499e-15, 5.8184e-14, 3.4648e-14, 8.5213e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.1495e-08, 1.0636e-09, 1.4008e-09, 4.6179e-09, 2.4393e-09, 8.8099e-09,\n 1.4654e-08, 1.8018e-08, 3.0678e-08, 2.2767e-09, 1.4918e-08, 1.1998e-09,\n 1.4163e-09, 5.8367e-09, 9.4961e-10, 3.9340e-09, 2.2946e-09, 1.2629e-08,\n 8.2201e-10, 8.9169e-09, 4.4152e-09, 5.5989e-09, 7.3190e-09, 3.0713e-09,\n 1.1878e-09, 1.2540e-08, 1.1017e-08, 4.1282e-09, 2.2685e-09, 3.0775e-09,\n 7.3156e-09, 6.5408e-09, 3.5800e-09, 5.8096e-09, 3.6308e-09, 1.2319e-08,\n 6.3221e-09, 1.3621e-09, 1.3584e-08, 6.4567e-09, 2.4772e-09, 1.8983e-09,\n 1.6335e-09, 8.3854e-09, 3.2760e-09, 1.4590e-09, 2.8502e-09, 1.0755e-09,\n 1.1244e-09, 6.1965e-09, 4.8208e-08, 6.1078e-09, 1.7308e-09, 3.2880e-09,\n 1.7205e-09, 7.7675e-09, 2.5046e-08, 3.1042e-09, 4.2851e-09, 6.2276e-09,\n 9.5270e-10, 2.3567e-08, 2.3917e-09, 1.3601e-09, 1.1006e-08, 1.0286e-08,\n 1.7887e-09, 7.3435e-09, 1.4914e-09, 1.4550e-08, 3.2026e-09, 1.7650e-09,\n 1.1077e-09, 3.7382e-09, 2.7461e-09, 1.3937e-08, 1.5372e-09, 2.8691e-09,\n 1.4543e-08, 1.4539e-09, 2.8424e-09, 1.1582e-09, 1.2051e-09, 3.2787e-09,\n 1.7057e-09, 1.1059e-08, 2.5344e-09, 1.6010e-09, 1.1493e-09, 3.0055e-09,\n 1.7844e-09, 1.3836e-09, 1.4456e-09, 2.5327e-09, 3.0169e-09, 4.0623e-08,\n 3.9775e-09, 4.8134e-09, 1.9056e-09, 2.8523e-09, 4.6182e-09, 7.0731e-09,\n 2.5237e-09, 1.0344e-09, 9.7569e-10, 7.6286e-09, 2.2921e-09, 2.4684e-09,\n 2.4512e-09, 2.2254e-09, 1.5978e-09, 2.3166e-09, 8.2303e-10, 6.6311e-09,\n 3.5302e-09, 1.4508e-09, 5.1760e-09, 7.0197e-10, 3.2949e-09, 1.8217e-09,\n 4.4780e-09, 3.2834e-09, 1.0687e-09, 6.3433e-09, 1.7812e-09, 2.0330e-09,\n 2.5002e-09, 3.5704e-09, 1.4949e-09, 1.8700e-09, 2.9118e-09, 2.4302e-09,\n 2.2365e-09, 1.3472e-09, 5.9160e-09, 8.9572e-10, 2.7672e-09, 9.1995e-09,\n 3.8455e-09, 2.5649e-09, 1.4144e-09, 2.6099e-09, 3.4185e-09, 1.2592e-09,\n 6.0475e-09, 1.4253e-09, 1.4535e-09, 4.4441e-09, 3.1234e-09, 1.2003e-09,\n 4.9185e-09, 2.0709e-08, 2.8781e-09, 1.0722e-08, 1.0235e-09, 1.4137e-08,\n 3.7391e-09, 2.8933e-09, 2.4566e-09, 7.1798e-09, 6.9186e-09, 9.0153e-10,\n 1.6344e-09, 5.8095e-09, 1.7185e-09, 2.1498e-09, 1.5099e-09, 2.4286e-09,\n 2.0426e-08, 9.0049e-09, 1.0825e-08, 8.0875e-10, 1.5731e-08, 2.0716e-09,\n 1.9581e-09, 2.9078e-09, 1.1515e-09, 5.4838e-09, 4.3004e-10, 3.1607e-09,\n 9.5625e-10, 9.1316e-10, 4.3254e-09, 2.0815e-09, 8.6916e-10, 1.9140e-08,\n 5.4858e-09, 4.4769e-09, 2.2577e-09, 5.2186e-09, 2.5185e-09, 2.1934e-09,\n 1.5492e-09, 6.7284e-09, 1.5641e-09, 4.6537e-10, 2.2038e-09, 2.4306e-09,\n 1.5175e-09, 7.2071e-09, 2.4095e-09, 4.2173e-10, 7.0869e-10, 4.0665e-09,\n 1.1820e-08, 4.0530e-09, 1.2201e-08, 1.0206e-09, 2.2838e-09, 5.8834e-09,\n 2.1235e-08, 3.5627e-09, 3.6983e-09, 1.7997e-09, 4.1025e-09, 1.5841e-08,\n 4.4979e-09, 5.0483e-09, 7.2531e-10, 4.3746e-10, 8.2633e-10, 7.1130e-09,\n 2.4986e-09, 4.1139e-09, 1.0904e-09, 2.4988e-09, 1.4148e-09, 1.2305e-09,\n 1.3594e-08, 1.5325e-09, 1.0589e-08, 3.0209e-09, 2.3333e-09, 2.9368e-09,\n 1.1853e-08, 6.0913e-09, 2.6563e-09, 3.6553e-09, 1.6188e-09, 1.5628e-09,\n 1.2187e-09, 2.4530e-09, 7.7072e-10, 1.3435e-09, 3.6993e-09, 2.4478e-09,\n 1.2743e-08, 6.1970e-09, 2.9127e-09, 1.6250e-09, 4.1276e-09, 5.0230e-09,\n 3.8303e-09, 2.4128e-09, 9.9516e-09, 1.2965e-09, 7.0157e-09, 9.8260e-09,\n 1.6419e-09, 9.7179e-09, 3.0627e-09, 2.5490e-09, 2.9788e-09, 7.8312e-10,\n 1.4273e-09, 2.4403e-09, 1.2178e-09, 4.9512e-09, 2.7999e-09, 2.6693e-09,\n 2.6694e-09, 4.2640e-09, 5.5620e-09, 2.4146e-09, 9.2410e-09, 2.6890e-09,\n 4.3004e-09, 6.4655e-09, 4.5257e-09, 2.5880e-09, 3.1017e-09, 1.2299e-09,\n 4.3101e-09, 1.1281e-09, 2.0336e-09, 4.0268e-09, 4.0532e-09, 2.8987e-08,\n 9.3857e-09, 8.2815e-10, 1.8702e-09, 2.3458e-09, 3.7964e-09, 8.3210e-09,\n 1.7890e-09, 2.3392e-09, 8.7259e-10, 6.9784e-10, 1.3856e-08, 3.2218e-09,\n 2.5614e-09, 1.2026e-08, 9.2797e-10, 1.9515e-09, 7.5823e-10, 3.0203e-08,\n 1.0898e-08, 1.2340e-09, 4.6220e-10, 1.8628e-09, 1.3041e-09, 4.8341e-09,\n 4.3645e-09, 1.8718e-09, 5.3845e-09, 3.0413e-09, 1.5569e-09, 1.0771e-09,\n 1.0372e-08, 1.3990e-09, 2.0281e-09, 3.4313e-09, 2.6976e-09, 1.4589e-08,\n 2.8508e-09, 1.1881e-08, 1.0171e-09, 9.3440e-09, 3.1789e-08, 3.7223e-09,\n 9.5785e-09, 1.4638e-08, 6.5355e-09, 4.1371e-09, 5.6147e-09, 6.9059e-09,\n 1.2926e-09, 3.0136e-09, 1.0655e-08, 1.0779e-09, 1.2142e-08, 1.1547e-09,\n 2.1649e-09, 3.5268e-09, 2.5977e-09, 7.5416e-10, 8.8670e-10, 1.2131e-08,\n 9.0672e-09, 1.1943e-09, 5.3064e-09, 1.0965e-08, 1.2186e-09, 1.7081e-08,\n 9.8414e-10, 2.3891e-08, 6.2279e-10, 4.5339e-09, 1.0536e-09, 1.2603e-09,\n 1.4080e-09, 1.0481e-08, 4.6743e-09, 1.0140e-08, 3.9791e-09, 1.7057e-09,\n 3.4150e-09, 1.1275e-09, 5.6958e-09, 3.9872e-09, 2.5916e-09, 3.5216e-09,\n 9.7124e-10, 6.9482e-09, 8.1337e-09, 2.4394e-09, 5.2976e-09, 2.4756e-09,\n 1.2093e-08, 3.1376e-09, 4.6710e-10, 3.4905e-09, 9.7178e-10, 3.4515e-09,\n 3.9076e-09, 1.2060e-08, 2.1963e-09, 2.0105e-09, 3.2678e-09, 2.9098e-09,\n 8.8421e-10, 8.7074e-09, 2.4169e-09, 5.2451e-09, 3.8509e-09, 1.0491e-09,\n 3.1111e-09, 2.8517e-09, 2.4955e-09, 4.6735e-09, 4.4863e-09, 3.4344e-09,\n 7.5698e-09, 2.9293e-09, 2.6669e-09, 9.4051e-10, 1.5073e-08, 6.6001e-09,\n 4.0951e-09, 3.2917e-09, 6.8877e-09, 3.0290e-09, 1.8683e-09, 5.1161e-09,\n 2.7322e-09, 1.4676e-08, 6.3362e-10, 7.7862e-10, 5.2755e-09, 8.8259e-09,\n 9.8470e-10, 1.2656e-09, 1.4682e-08, 1.9462e-09, 5.4928e-09, 3.3843e-09,\n 2.2400e-09, 1.5246e-08, 1.5965e-09, 6.9172e-09, 1.5781e-09, 2.7563e-09,\n 9.3893e-10, 1.3197e-09, 3.3107e-09, 3.1495e-09, 8.4930e-10, 6.3136e-09,\n 2.4511e-09, 8.9524e-09, 9.9239e-09, 1.1053e-08, 1.8873e-09, 2.0116e-09,\n 1.0416e-08, 9.4569e-10, 2.1822e-09, 6.9605e-09, 1.9986e-09, 2.3844e-09,\n 7.4268e-09, 6.3456e-09, 8.3442e-10, 1.0080e-09, 1.1925e-08, 5.2269e-09,\n 1.5941e-09, 5.7949e-09, 2.1404e-09, 1.3224e-09, 2.2225e-08, 1.6139e-09,\n 4.4315e-09, 2.5816e-09, 2.4877e-09, 3.9943e-09, 7.9959e-09, 1.3723e-09,\n 1.3264e-09, 3.6499e-09, 1.7952e-09, 1.7327e-09, 6.7730e-09, 1.2454e-09,\n 2.1964e-08, 2.9586e-09, 1.1852e-08, 2.7932e-09, 2.8042e-09, 2.4755e-09,\n 2.7584e-09, 1.8254e-08, 6.3076e-09, 1.6398e-09, 6.9684e-09, 4.3283e-09,\n 2.3688e-09, 1.4374e-08, 1.4180e-09, 1.1418e-09, 5.1955e-09, 1.7850e-08,\n 5.8280e-09, 5.9601e-10, 2.8484e-09, 7.4049e-09, 7.9195e-10, 1.6152e-09,\n 3.1897e-09, 2.9110e-09, 2.1086e-09, 9.9382e-10, 5.4838e-09, 1.8142e-08,\n 4.2933e-09, 2.2058e-09, 3.1111e-09, 8.0730e-09, 6.8462e-09, 9.8924e-10,\n 2.9634e-09, 2.5168e-09, 3.3588e-09, 6.6393e-09, 4.8443e-09, 2.7079e-09,\n 2.7249e-09, 2.0020e-09, 3.6580e-09, 3.0726e-09, 2.1203e-09, 3.4762e-08,\n 6.9488e-10, 1.3675e-09, 2.1633e-09, 7.7895e-10, 3.8761e-09, 1.2991e-09,\n 1.2172e-08, 2.0401e-09, 2.0107e-09, 5.1475e-09, 8.2960e-09, 2.3432e-09,\n 1.9012e-09, 1.5000e-08, 1.4987e-09, 6.1275e-09, 2.2601e-09, 1.1578e-09,\n 2.1186e-08, 4.8988e-09, 3.3225e-09, 1.9109e-08, 1.2743e-08, 1.0626e-08,\n 2.3193e-09, 4.4403e-09, 4.0196e-09, 1.6283e-09, 9.1083e-09, 5.6324e-10,\n 8.5609e-10, 1.0766e-08, 1.0359e-08, 2.0032e-08, 6.3304e-09, 1.0849e-08,\n 1.9356e-08, 9.8572e-09, 5.1036e-09, 6.7015e-09, 1.1957e-09, 1.7832e-08,\n 9.9728e-09, 7.0540e-09, 4.1351e-09, 2.9759e-09, 1.8183e-09, 2.1511e-09,\n 9.6837e-09, 9.1395e-09, 2.1290e-08, 2.3178e-09, 2.1716e-09, 8.8149e-09,\n 2.6339e-09, 4.6572e-09, 1.4640e-09, 8.9458e-10, 8.3068e-09, 2.0999e-09,\n 2.7277e-09, 1.1208e-08, 5.8602e-10, 2.9431e-09, 1.5574e-09, 1.8774e-09,\n 9.8161e-10, 6.5960e-09, 8.0295e-09, 1.9395e-09, 4.9315e-09, 8.6387e-10,\n 4.7113e-09, 1.6968e-09, 1.1659e-09, 3.9276e-09, 1.6331e-08, 2.3668e-08,\n 4.9386e-09, 5.0012e-09, 1.1480e-09, 2.9475e-09, 3.3319e-09, 1.8825e-09,\n 2.1517e-09, 1.4124e-08, 7.4159e-09, 1.1001e-09, 4.1289e-09, 5.9569e-09,\n 7.3343e-09, 3.4437e-09, 1.0282e-09, 1.8904e-09, 1.0298e-08, 2.0129e-08,\n 2.4181e-09, 2.0010e-09, 2.4877e-09, 3.9375e-09, 5.7209e-09, 2.8479e-09,\n 3.7290e-09, 2.2868e-09, 9.7164e-09, 3.3148e-09, 1.7302e-09, 2.4554e-09,\n 2.1915e-09, 1.1115e-08, 1.0207e-08, 3.6830e-09, 5.3049e-09, 1.6432e-09,\n 2.3327e-08, 1.5872e-09, 1.4484e-09, 1.2436e-08], device='cuda:0')" + }, + "66": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.5800e-36, 3.0059e-36, -1.8486e-36, ..., -1.2270e-36,\n -2.9565e-36, -3.6698e-36],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n ...,\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.0444e-08, 1.6787e-07, 6.8019e-09, ..., 3.1697e-07, 6.3293e-07,\n 3.1393e-08],\n [1.0012e-09, 5.0445e-09, 6.5672e-10, ..., 1.0730e-08, 1.6282e-08,\n 2.0312e-09],\n [4.0408e-08, 1.4852e-07, 4.0220e-09, ..., 2.7066e-07, 5.6956e-07,\n 2.6870e-08],\n ...,\n [1.1442e-08, 4.9385e-09, 2.2204e-09, ..., 1.9835e-09, 1.4605e-09,\n 1.5479e-08],\n [8.0570e-08, 3.4126e-08, 3.3863e-09, ..., 2.9292e-08, 1.7506e-08,\n 8.2953e-08],\n [5.4795e-08, 2.5532e-08, 4.5414e-09, ..., 1.7223e-08, 1.5643e-08,\n 5.9332e-08]], device='cuda:0')" + }, + "67": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 8.4331e-37, 5.6052e-45, -5.6052e-45, -4.3131e-13, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45], device='cuda:0')", + "exp_avg_sq": "tensor([1.1700e-06, 5.1851e-08, 9.1352e-07, 4.9986e-08, 5.2239e-08, 1.0197e-08,\n 4.0272e-08, 4.1949e-08], device='cuda:0')" + }, + "68": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.5705e-36, 2.9949e-36, -1.8418e-36, ..., -1.2225e-36,\n -2.9457e-36, -3.6563e-36],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n ...,\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.0455e-08, 1.6822e-07, 6.8074e-09, ..., 3.1765e-07, 6.3445e-07,\n 3.1353e-08],\n [1.0011e-09, 5.0463e-09, 6.5567e-10, ..., 1.0734e-08, 1.6288e-08,\n 2.0321e-09],\n [4.0511e-08, 1.4888e-07, 4.0317e-09, ..., 2.7132e-07, 5.7103e-07,\n 2.6924e-08],\n ...,\n [1.1372e-08, 4.9076e-09, 2.2073e-09, ..., 1.9755e-09, 1.4541e-09,\n 1.5384e-08],\n [8.0547e-08, 3.4116e-08, 3.3813e-09, ..., 2.9299e-08, 1.7497e-08,\n 8.2917e-08],\n [5.4744e-08, 2.5501e-08, 4.5298e-09, ..., 1.7217e-08, 1.5618e-08,\n 5.9265e-08]], device='cuda:0')" + }, + "69": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 8.4038e-37, 5.6052e-45, -5.6052e-45, -4.2981e-13, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45], device='cuda:0')", + "exp_avg_sq": "tensor([1.1725e-06, 5.1846e-08, 9.1578e-07, 5.0083e-08, 5.2278e-08, 1.0173e-08,\n 4.0270e-08, 4.1911e-08], device='cuda:0')" + }, + "70": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 2.5820e-36, 3.0083e-36, -1.8500e-36, ..., -1.2280e-36,\n -2.9589e-36, -3.6727e-36],\n [ 5.6052e-45, -5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n ...,\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.0508e-08, 1.6824e-07, 6.8133e-09, ..., 3.1765e-07, 6.3447e-07,\n 3.1453e-08],\n [1.0023e-09, 5.0515e-09, 6.5705e-10, ..., 1.0745e-08, 1.6306e-08,\n 2.0342e-09],\n [4.0548e-08, 1.4887e-07, 4.0315e-09, ..., 2.7129e-07, 5.7100e-07,\n 2.6949e-08],\n ...,\n [1.1466e-08, 4.9479e-09, 2.2257e-09, ..., 1.9871e-09, 1.4654e-09,\n 1.5514e-08],\n [8.0836e-08, 3.4238e-08, 3.3981e-09, ..., 2.9381e-08, 1.7565e-08,\n 8.3229e-08],\n [5.4987e-08, 2.5621e-08, 4.5569e-09, ..., 1.7278e-08, 1.5697e-08,\n 5.9542e-08]], device='cuda:0')" + }, + "71": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 8.4412e-37, 5.6052e-45, -5.6052e-45, -4.3172e-13, -5.6052e-45,\n 5.6052e-45, -5.6052e-45, 5.6052e-45], device='cuda:0')", + "exp_avg_sq": "tensor([1.1725e-06, 5.1911e-08, 9.1580e-07, 5.0141e-08, 5.2379e-08, 1.0216e-08,\n 4.0381e-08, 4.2057e-08], device='cuda:0')" + } + }, + "param_groups": [ + { + "lr": 0.0009755527298894294, + "name": "shared", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 0, + 1 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_384", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 2, + 3, + 4 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_512", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 5, + 6, + 7 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_768", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 8, + 9, + 10 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_1024", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 11, + 12, + 13 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_1280", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 14, + 15, + 16 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_1536", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 17, + 18, + 19 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_1792", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 20, + 21, + 22 + ] + }, + { + "lr": 0.0009755527298894294, + "name": "scale_2048", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 23, + 24, + 25 + ] + }, + { + "lr": 0.00048778860081564085, + "name": "fusion", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0005, + "params": [ + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + } + ] + }, + "scheduler_state_dict": { + "T_0": 10, + "T_i": 10, + "T_mult": 2, + "eta_min": 1e-06, + "T_cur": 1, + "base_lrs": [ + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.0005 + ], + "last_epoch": 1, + "_step_count": 0, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.0009755527298894294, + 0.00048778860081564085 + ] + }, + "metrics": { + "best_val_acc": 83.662, + "best_epoch": 0, + "scale_accuracies": { + "384": 83.27, + "512": 83.312, + "768": 83.59, + "1024": 83.496, + "1280": 83.662, + "1536": 83.514, + "1792": 83.398, + "2048": 83.598 + }, + "training_history": { + "epochs": [ + 1 + ], + "train_loss": [ + 1.8410232910713829 + ], + "train_acc": [ + 82.10982643168299 + ], + "val_acc": [ + 83.662 + ], + "scale_accs": { + "384": [ + 83.27 + ], + "512": [ + 83.312 + ], + "768": [ + 83.59 + ], + "1024": [ + 83.496 + ], + "1280": [ + 83.662 + ], + "1536": [ + 83.514 + ], + "1792": [ + 83.398 + ], + "2048": [ + 83.598 + ] + }, + "lr": [ + 0.0009755527298894294 + ] + } + }, + "train_config": { + "name": "david_training", + "run_id": "20251012_181608", + "dataset_name": "AbstractPhil/imagenet-clip-features-orderly", + "model_variant": "clip_vit_laion_bigg14", + "num_classes": 1000, + "preset": "clip_vit_bigg14", + "custom_config_path": null, + "num_classes_override": null, + "use_belly_override": null, + "belly_expand_override": null, + "progressive_training_override": false, + "scale_warmup_epochs_override": null, + "num_epochs": 10, + "batch_size": 1024, + "learning_rate": 0.001, + "weight_decay": 1e-05, + "warmup_epochs": 3, + "use_rose_loss": false, + "rose_initial_weight": 0.1, + "rose_max_weight": 0.5, + "rose_weight_schedule": "adaptive", + "use_cayley_loss": false, + "cayley_weight": 0.001, + "scale_loss_balance": null, + "use_mixed_precision": false, + "gradient_clip": 10.0, + "scheduler_type": "cosine_restarts", + "min_lr": 1e-06, + "freeze_strategy": "none", + "freeze_threshold": 90.0, + "unfreeze_on_plateau": true, + "patience": 10, + "track_gradients": true, + "gradient_scale_threshold": 1e-05, + "gradient_scale_multiplier": 10.0, + "log_interval": 50, + "val_interval": 1, + "save_interval": 5, + "log_fusion_weights": true, + "log_loss_components": true, + "save_format": "safetensors", + "hf_repo": "AbstractPhil/gated-david", + "upload_to_hub": true, + "base_dir": "./david_training", + "num_workers": 10, + "pin_memory": true, + "prefetch_factor": 4, + "persistent_workers": true + } +} \ No newline at end of file