diff --git "a/weights/David-partial_shared-deep_efficiency/20251012_161107/best_model_acc84.59_metadata.json" "b/weights/David-partial_shared-deep_efficiency/20251012_161107/best_model_acc84.59_metadata.json" new file mode 100644--- /dev/null +++ "b/weights/David-partial_shared-deep_efficiency/20251012_161107/best_model_acc84.59_metadata.json" @@ -0,0 +1,856 @@ +{ + "epoch": 6, + "optimizer_state_dict": { + "state": { + "0": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-2.2419e-04, 9.7594e-05, 1.9321e-04, ..., -1.7080e-04,\n -2.3580e-04, -1.7908e-04],\n [-7.0754e-05, -9.3149e-05, -5.1123e-05, ..., -5.2656e-05,\n 5.3606e-05, -5.2034e-05],\n ...,\n [-1.4787e-04, -9.5923e-05, -1.9313e-04, ..., 6.4061e-05,\n 2.8698e-04, -1.9341e-05],\n [ 8.7772e-05, -4.8194e-05, 3.2184e-05, ..., -8.2740e-06,\n -1.3482e-04, -9.6444e-05],\n [ 9.3488e-05, 5.0295e-05, -1.0247e-05, ..., 4.1310e-05,\n 1.6709e-04, -8.1603e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.6742e-14, 7.4663e-14, 1.1810e-13, ..., 1.1622e-14, 4.3209e-14,\n 1.0016e-14],\n [1.2068e-07, 8.1700e-08, 1.2592e-07, ..., 1.0669e-07, 2.9411e-07,\n 1.1145e-07],\n [7.9889e-08, 8.5904e-08, 9.2354e-08, ..., 1.3127e-07, 1.6006e-07,\n 8.9640e-08],\n ...,\n [1.1025e-07, 7.6216e-08, 8.9117e-08, ..., 1.7570e-07, 2.0899e-07,\n 1.0879e-07],\n [1.3897e-07, 1.3124e-07, 1.5568e-07, ..., 1.6070e-07, 3.3647e-07,\n 1.4245e-07],\n [5.1004e-08, 4.6887e-08, 6.0946e-08, ..., 6.8651e-08, 1.0174e-07,\n 5.6656e-08]], device='cuda:0')" + }, + "1": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 5.6052e-45, 7.4261e-03, 1.7443e-03, ..., 8.7529e-03,\n -2.2132e-03, 3.0179e-03], device='cuda:0')", + "exp_avg_sq": "tensor([1.6533e-10, 2.6364e-04, 2.5546e-04, ..., 2.8632e-04, 4.0242e-04,\n 1.3336e-04], device='cuda:0')" + }, + "2": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -5.7021e-06, -2.3415e-07, ..., -4.9317e-06,\n -6.8237e-06, -3.9364e-06],\n [ 5.6052e-45, 6.0811e-06, 1.7260e-08, ..., 3.7743e-07,\n -3.0345e-06, -5.7763e-07],\n [-5.6052e-45, 9.7676e-06, 1.3462e-06, ..., -4.4020e-06,\n 1.8569e-05, 1.3613e-08],\n ...,\n [ 5.6052e-45, -4.8390e-09, -1.1039e-05, ..., -9.5039e-06,\n 4.1537e-06, 1.2588e-04],\n [-5.6052e-45, 1.7156e-05, -7.9514e-06, ..., 1.0467e-06,\n 1.5602e-05, -1.3900e-05],\n [ 5.6052e-45, 3.1337e-06, 9.0376e-06, ..., 7.8251e-06,\n 4.5636e-06, -2.1487e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.2504e-13, 8.3534e-10, 3.9309e-11, ..., 3.9391e-10, 1.2786e-09,\n 4.6096e-10],\n [1.9946e-13, 3.8841e-09, 1.3075e-10, ..., 1.0613e-10, 5.0842e-10,\n 1.5428e-10],\n [6.0927e-13, 1.5108e-09, 3.8170e-10, ..., 2.6101e-10, 1.1392e-09,\n 5.3189e-11],\n ...,\n [4.5365e-13, 2.0812e-09, 1.9009e-10, ..., 7.9477e-09, 1.8061e-09,\n 3.7587e-09],\n [1.0478e-13, 3.1057e-09, 1.0841e-09, ..., 2.1496e-10, 1.1079e-09,\n 6.2350e-10],\n [1.4713e-13, 6.4219e-10, 1.9826e-10, ..., 1.3628e-09, 1.2759e-09,\n 2.7144e-10]], device='cuda:0')" + }, + "3": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 7.0880e-04, -3.3977e-04, -1.9342e-04, -1.1043e-05, 6.2506e-06,\n -3.8578e-04, 9.8100e-05, -3.7047e-04, -4.2252e-04, -5.5212e-04,\n 5.8178e-04, -3.6452e-04, -3.8304e-04, -1.3862e-04, -5.2748e-04,\n 4.3041e-04, 4.4350e-04, -6.1983e-04, -4.0533e-05, -9.4890e-05,\n -4.8817e-04, 1.0563e-03, -8.2632e-04, 5.6052e-45, 1.8011e-03,\n 1.3060e-03, 1.7025e-04, -5.6187e-04, -2.8553e-04, -3.0256e-04,\n -4.3473e-04, 3.5431e-04, -6.5646e-04, 1.4475e-04, 1.3332e-04,\n 3.5887e-04, 4.6069e-05, 1.0955e-03, 2.1221e-05, -9.3651e-04,\n -1.4631e-03, 7.3204e-05, 5.9443e-05, -9.6472e-05, -3.3450e-04,\n -7.7868e-05, 3.8283e-04, -2.2253e-09, -8.3843e-05, -2.6266e-03,\n 4.5324e-04, 2.2458e-04, 5.7344e-04, -2.7156e-04, -1.8643e-04,\n -9.7515e-05, -9.7207e-04, -2.4284e-04, 4.2404e-04, 3.4450e-04,\n 8.1556e-04, 2.2197e-04, 6.9497e-04, -2.0717e-04, 8.1892e-04,\n -5.2542e-04, -1.1041e-04, -2.7650e-04, -1.0211e-03, 9.8216e-05,\n -1.7102e-04, 3.5268e-04, -6.9201e-04, -2.4173e-04, 1.9789e-04,\n 5.6052e-45, -3.2577e-04, 9.4430e-05, -6.8490e-04, -3.6734e-04,\n -3.5668e-04, -6.9729e-04, -1.2289e-05, 7.9067e-04, 5.6052e-45,\n 5.2308e-04, -8.1844e-04, 5.3810e-04, 4.3952e-04, 3.8034e-04,\n 6.7093e-04, 6.0656e-04, -3.8070e-04, 1.6978e-03, -2.2221e-05,\n 7.8438e-04, -7.7281e-05, -6.4046e-04, 1.3008e-03, 6.8994e-05,\n 7.7802e-04, 6.2013e-04, 5.5359e-05, -5.5000e-04, 1.0931e-03,\n -7.8717e-05, 3.9390e-05, 7.5150e-04, -3.9225e-04, 1.5621e-03,\n -7.7154e-05, 3.8689e-04, 5.0188e-04, 3.7882e-04, 8.1017e-04,\n 1.3974e-04, -2.4792e-04, -1.1259e-04, 5.0557e-04, -3.3669e-04,\n -2.8878e-03, 5.2369e-04, -1.4926e-04, 6.4545e-04, -5.6016e-04,\n -8.4259e-04, 8.9402e-04, -7.9551e-04, -8.9802e-04, -6.0811e-04,\n -1.9822e-04, -3.0910e-04, -7.1410e-04, 8.4766e-05, -1.4750e-04,\n 7.9367e-05, -3.0978e-04, -7.9630e-04, -8.1904e-04, -5.2135e-04,\n 1.3016e-04, -8.0369e-05, -8.5791e-04, -1.7167e-04, -1.4945e-04,\n -4.0359e-04, 6.0900e-04, -5.1997e-04, 4.1940e-04, -3.3010e-04,\n 4.5094e-05, -2.2981e-04, -5.3634e-04, -4.5873e-05, -1.0016e-03,\n -3.5645e-04, 2.9468e-04, 3.2327e-05, 1.5348e-04, 1.1179e-04,\n 2.5912e-04, 3.4932e-04, -1.2642e-05, -5.4962e-04, 5.6052e-45,\n -1.0985e-03, -2.8942e-04, 3.1151e-04, -4.4817e-04, 1.6221e-04,\n -3.7968e-04, 1.2733e-04, -3.5254e-04, 4.2316e-04, -1.2783e-04,\n 4.6739e-04, -2.8524e-04, -1.6282e-03, -1.5907e-03, -5.4054e-04,\n 1.2668e-03, -5.5885e-05, -8.9482e-05, 5.2613e-05, 6.8796e-04,\n -5.1906e-05, -3.7064e-04, -3.6187e-04, -3.5715e-04, 1.6375e-04,\n 1.0283e-03, 4.5682e-04, 8.1875e-05, -3.0876e-04, 8.3333e-04,\n 2.1065e-04, -4.3030e-04, -2.8639e-04, 8.3972e-04, -7.2494e-04,\n -6.6225e-06, 7.1537e-04, 2.0178e-04, 1.1577e-03, 1.4667e-03,\n -6.6048e-04, 7.0669e-05, 7.6706e-05, -1.6542e-03, -2.8533e-04,\n -7.6440e-04, -2.9583e-04, 4.1026e-04, -2.3764e-04, 6.9626e-04,\n 1.7840e-04, 1.2335e-05, 5.2314e-04, -6.4714e-04, 3.0440e-04,\n -1.0294e-03, 5.6052e-45, 5.1587e-04, -1.0717e-04, -3.4333e-04,\n -5.3013e-05, -1.8029e-03, 1.0921e-04, -8.0346e-04, -2.2021e-05,\n 5.2243e-04, -4.7227e-04, 7.5044e-04, -6.0855e-04, 4.8762e-04,\n 6.5130e-04, 5.4184e-04, -3.2145e-04, 6.8564e-04, -6.3812e-04,\n -4.2213e-04, 6.0506e-04, 2.7993e-04, -5.1638e-04, 9.9199e-05,\n 2.2500e-04, -5.7954e-04, -5.4214e-04, -2.7246e-04, -2.5718e-05,\n 8.2512e-05, -3.4832e-04, -9.6906e-05, 3.1101e-06, 6.6985e-04,\n -2.5164e-04, 7.0060e-04, 1.1983e-03, -3.0169e-04, -4.7771e-04,\n -9.8218e-04, 3.7499e-04, 4.5382e-04, -3.0104e-04, -9.8962e-04,\n -9.0385e-04, -1.1703e-03, 1.1094e-03, 4.0559e-04, 2.7495e-04,\n 5.7840e-04, 2.2083e-06, -1.1970e-04, 1.4497e-04, 7.8854e-04,\n 5.5630e-04, -1.8927e-04, 2.9090e-04, 9.8009e-04, 2.0495e-04,\n -1.2290e-05, 3.4694e-04, 1.9426e-03, -1.4835e-04, 6.3813e-04,\n -7.5648e-05, 6.4876e-05, -9.9592e-04, 5.9164e-05, 5.6052e-45,\n -2.6749e-04, -1.2175e-04, -1.0349e-03, -4.1245e-04, -8.8992e-04,\n 5.2610e-04, -1.6948e-03, 4.8318e-04, 4.5533e-04, 4.6116e-04,\n -4.0023e-04, 2.9232e-04, -3.0989e-05, 6.4029e-05, 2.9921e-04,\n 1.6128e-04, -2.7078e-04, -2.5608e-04, -2.6338e-05, -1.2732e-03,\n -1.0655e-06, 3.4630e-04, 1.1287e-03, -2.4466e-04, 9.1901e-05,\n -3.9765e-04, -5.3004e-04, 3.4831e-04, 3.9871e-16, 4.9735e-04,\n -1.5983e-04, 6.2289e-04, 2.2358e-04, -3.4950e-04, -1.1100e-05,\n 1.2476e-03, 9.6527e-04, -1.4155e-04, 3.4106e-04, 5.0606e-04,\n 1.2637e-04, 9.9166e-05, 3.2135e-04, -1.9164e-03, -4.2586e-05,\n 1.5937e-04, 1.9739e-04, 3.0344e-04, -3.3246e-04, 8.1678e-04,\n 1.3907e-04, -8.3719e-04, 1.5376e-05, -1.3380e-03, -4.9564e-05,\n -6.5045e-04, 8.7022e-04, -7.5924e-04, -8.6839e-05, -7.2735e-05,\n 4.0788e-04, 4.3953e-04, 4.7426e-04, -1.1033e-04, 5.3444e-04,\n -3.3652e-04, -4.6071e-04, -2.7731e-04, -6.4301e-31, 4.4365e-04,\n 2.6885e-04, -9.1304e-04, -2.2482e-04, -2.0666e-04, 9.0694e-04,\n 1.1536e-05, 4.6741e-05, 7.2714e-04, -7.8548e-04, 7.3492e-04,\n 1.2938e-04, 8.8201e-04, -4.0259e-04, -3.4589e-04, 5.2704e-04,\n 1.0833e-03, 2.6565e-04, 1.9879e-04, -3.2439e-05, -4.7473e-04,\n 8.9184e-04, 5.6052e-45, 9.8289e-04, -7.8338e-05, 5.0753e-06,\n 1.7295e-06, 5.2593e-04, -4.1908e-05, 7.1295e-04, -2.5273e-04,\n -1.2549e-03, 4.0771e-05, 3.6888e-04, -1.3908e-04, 3.8320e-04,\n -1.9215e-03, 3.0944e-04, -2.1885e-04, -8.0093e-05, 6.3317e-04,\n 5.6052e-45, 9.5380e-05, -2.1360e-04, 1.5784e-04, 2.1837e-04,\n -1.7541e-04, 9.4031e-04, 4.6731e-04, 5.6052e-45, 3.4004e-04,\n 1.1207e-04, 1.9517e-04, 7.4396e-04, 2.1587e-04, 6.9095e-06,\n 6.2702e-05, 2.6089e-03, -1.1744e-03, 1.3512e-04, 4.4574e-04,\n 4.8001e-04, 2.2564e-04, 4.1123e-04, 8.0702e-04, -6.7419e-04,\n 5.7672e-04, -2.9026e-04, 6.3216e-04, -3.6661e-04, -2.5170e-04,\n -3.7934e-04, -4.4198e-04, 8.3642e-05, 1.8096e-04, -3.9296e-04,\n -6.5004e-04, -1.6885e-04, -5.6219e-04, -4.0210e-05, 5.7198e-30,\n -1.7922e-03, 6.6382e-04, 9.7729e-04, 2.2858e-04, -2.4107e-04,\n 6.4350e-04, 5.6052e-45, -1.3493e-03, 2.3279e-04, 1.3135e-04,\n -1.8101e-04, -4.9820e-04, 2.8153e-04, 7.9024e-04, -2.8828e-04,\n -2.2081e-04, 4.5016e-04, -3.2879e-05, 5.6052e-45, 6.7661e-05,\n 5.0447e-04, -7.4525e-04, 2.3625e-04, 1.1498e-04, -1.2748e-04,\n -4.3345e-04, -1.2050e-03, -1.4561e-04, 1.2247e-35, 5.5846e-05,\n -2.5843e-04, -1.1225e-04, -3.2769e-04, 4.9131e-04, -4.8652e-04,\n -8.2688e-05, 8.1306e-04, -4.0964e-04, -1.7199e-03, -7.2048e-04,\n -1.9280e-04, 3.2149e-05, 5.6052e-45, 5.6474e-05, 4.3345e-04,\n 8.1755e-04, -1.3303e-04, -4.3591e-04, 9.8173e-05, 2.5322e-04,\n 2.4284e-04, 3.4759e-04, 9.1118e-04, 9.0866e-04, 1.0347e-04,\n 3.6491e-04, -5.2169e-04, 2.6080e-04, 1.0974e-04, 1.5481e-03,\n 4.5699e-04, -5.6886e-04, -2.0966e-04, -5.6701e-04, 3.5231e-04,\n -5.0942e-04, 3.2761e-04, -1.4497e-04, -7.4719e-04, 6.7787e-04,\n 9.4397e-04, 3.9430e-04, -1.0210e-03, 8.7795e-04, 3.2453e-04,\n 1.2159e-03, -2.7000e-03, 4.3515e-05, 4.1463e-04, -5.7222e-05,\n 1.4911e-03, 1.0832e-04, -3.9171e-04, -1.6532e-04, 1.0213e-04,\n -1.5718e-04, -2.0181e-04, -1.6047e-04, -3.6812e-04, 7.9212e-04,\n 7.5941e-04, 3.0821e-04, -7.6750e-04, 3.8789e-04, -2.6979e-04,\n -5.6200e-04, 5.3132e-04, 6.5017e-04, -1.0131e-04, -2.2544e-04,\n -4.0251e-04, 6.0153e-04, -2.2518e-04, -1.0781e-04, -1.3101e-03,\n -9.8414e-04, -7.2309e-04, -1.9746e-04, 6.9010e-05, -5.2223e-04,\n -1.1700e-03, -1.7594e-04, -1.2439e-04, -1.7882e-04, -5.8371e-04,\n -8.3835e-06, 1.6887e-03, -9.4422e-06, 2.6129e-05, 9.7521e-04,\n 3.6481e-04, -1.8436e-04, 2.6218e-04, -5.7865e-04, 5.6514e-04,\n 4.6302e-04, -7.8618e-04, 4.6794e-05, -3.3591e-04, 2.1957e-04,\n 1.6881e-04, 4.1952e-04, -2.2145e-03, -1.3915e-04, -2.9043e-03,\n -4.6318e-04, -1.0187e-03, 1.9931e-04, -7.9062e-04, -6.1937e-04,\n 1.9703e-04, -1.7599e-04, 1.4094e-04, 7.2016e-05, -8.6668e-05,\n 4.8146e-04, 7.4275e-04, -2.2730e-03, 1.1118e-07, 1.9840e-04,\n 2.7882e-04, -2.7069e-05, 1.1704e-04, 1.9654e-03, 8.9414e-04,\n 2.9887e-05, 8.0908e-05, -2.6434e-04, -1.0694e-03, -5.0127e-04,\n 8.6724e-04, 2.2823e-04, -1.1243e-03, -4.9134e-04, -3.4915e-04,\n -7.0849e-04, 2.3613e-04, 4.6350e-04, 2.2251e-04, -2.4590e-04,\n 5.9704e-05, -1.2977e-03, 2.7567e-04, 4.0844e-04, -3.3647e-04,\n 6.5795e-04, -6.7379e-05, -7.0776e-05, 1.9481e-04, -5.6052e-45,\n 3.9244e-04, -4.3920e-04, -1.5060e-03, -1.4284e-03, -1.3849e-03,\n 8.2665e-05, 8.5339e-05, 1.2212e-03, 8.0516e-05, -5.5534e-04,\n -4.5035e-04, 3.2909e-04, -1.0843e-04, -4.9114e-04, -4.6597e-04,\n 3.6308e-04, -4.6248e-04, 2.7633e-04, 1.7007e-04, 8.7042e-05,\n 1.0324e-03, 2.4342e-04, -1.0236e-03, 6.4563e-04, 3.9842e-05,\n -7.9087e-04, 4.0063e-04, 1.0510e-04, -1.7517e-03, -2.4388e-04,\n -4.7089e-04, 4.5164e-04, -1.8677e-05, 5.5764e-04, -2.8088e-04,\n -4.3681e-04, 4.5962e-04, -1.5007e-04, 6.1373e-04, -2.2218e-05,\n -6.7269e-04, 2.2553e-05, 2.4019e-05, -2.2661e-04, -6.2239e-04,\n 4.4421e-04, 3.2185e-04, -7.8082e-04, 2.7749e-04, 2.0321e-04,\n 9.9953e-04, 1.0187e-04, 8.5053e-04, 1.1096e-03, 4.2949e-05,\n -9.9701e-04, 1.3933e-03, -4.8334e-04, 7.1786e-04, -2.3281e-03,\n -4.2224e-04, -1.5307e-03, -3.8455e-04, 4.1875e-04, 1.1872e-04,\n 2.3468e-04, 1.4340e-03, 6.9543e-04, -5.8112e-04, -2.8734e-04,\n 7.5944e-04, 2.2594e-04, 6.6743e-04, 2.7404e-04, -3.2631e-04,\n 4.7397e-04, 2.6108e-04, -1.1558e-05, 9.5211e-04, -3.9959e-04,\n -6.1173e-04, 5.3980e-04, -1.2462e-04, 4.9517e-04, 6.0092e-04,\n 9.4036e-04, 5.1194e-06, -6.1695e-04, 7.6287e-04, 8.4506e-04,\n 9.1750e-05, 6.5517e-04, 4.2987e-06, -7.4389e-04, -4.5484e-04,\n -3.8044e-04, -1.5759e-05, 1.3301e-03, -2.0461e-04, 5.3469e-04,\n -5.5649e-04, 2.7242e-05, -2.6999e-04, -3.9657e-04, 1.9267e-04,\n -5.5285e-04, 1.3384e-04, 2.2631e-04, -2.8546e-04, -4.3099e-04,\n -1.0642e-03, -9.1450e-04, 1.7626e-03, -2.2067e-03, 5.0355e-04,\n 8.7132e-04, -6.2645e-04, 1.7390e-05, -3.9074e-04, -7.7537e-04,\n 9.1999e-05, -1.0724e-03, 1.7213e-04, 3.8077e-04, -3.2279e-04,\n -2.3591e-04, -5.1096e-04, -5.1272e-05, -9.8692e-05, -3.8590e-04,\n -6.3270e-04, -2.5579e-05, -6.0404e-04, 1.5332e-04, 2.6848e-05,\n -2.5293e-04, -8.7055e-04, 4.1540e-04, -5.9016e-04, -1.4010e-05,\n 1.1383e-04, -1.7288e-04, -2.5654e-04, -6.4268e-04, -6.5710e-04,\n 1.0724e-03, -4.6625e-05, 1.8249e-04], device='cuda:0')", + "exp_avg_sq": "tensor([4.2462e-06, 3.9467e-06, 2.0461e-06, 4.3662e-06, 3.7599e-06, 3.6340e-06,\n 3.3728e-06, 3.2358e-06, 3.4156e-06, 3.4007e-06, 4.2036e-06, 3.0102e-06,\n 3.1626e-06, 3.9195e-06, 4.1641e-06, 3.0909e-06, 2.7776e-06, 5.7534e-06,\n 9.3235e-07, 3.0946e-06, 4.4176e-06, 3.9219e-06, 3.3392e-06, 7.0795e-12,\n 3.3090e-06, 4.2607e-06, 3.4358e-06, 3.6663e-06, 3.5881e-06, 2.8301e-06,\n 3.4154e-06, 4.7718e-06, 3.7604e-06, 4.1198e-06, 3.1096e-06, 3.2751e-06,\n 4.3858e-06, 4.5576e-06, 3.9747e-06, 4.4321e-06, 3.6271e-06, 2.9158e-06,\n 3.7167e-06, 3.3333e-06, 2.9841e-06, 3.9236e-06, 1.9692e-06, 7.7147e-10,\n 3.4676e-06, 3.9647e-06, 4.1530e-06, 4.1965e-06, 4.0774e-06, 4.6910e-06,\n 3.1951e-06, 3.5669e-06, 3.4849e-06, 3.3999e-06, 4.0715e-06, 4.0603e-06,\n 4.2276e-06, 4.3662e-06, 3.7099e-06, 3.1426e-06, 2.9824e-06, 3.1589e-06,\n 4.2841e-06, 3.6507e-06, 2.9362e-06, 3.5794e-06, 3.5705e-06, 3.7795e-06,\n 4.0950e-06, 4.2623e-06, 4.3699e-06, 1.2157e-10, 3.5671e-06, 3.1777e-06,\n 4.1976e-06, 9.7523e-07, 6.1712e-06, 3.6175e-06, 3.5039e-06, 4.7496e-06,\n 2.5196e-12, 3.0294e-06, 3.1726e-06, 3.0281e-06, 1.5223e-06, 3.6378e-06,\n 4.3502e-06, 4.9147e-06, 3.4392e-06, 4.6121e-06, 3.4325e-06, 4.2054e-06,\n 2.3942e-06, 4.1175e-06, 3.4107e-06, 3.7836e-06, 4.3866e-06, 3.4880e-06,\n 3.6820e-06, 3.2312e-06, 3.4281e-06, 5.2853e-06, 4.8647e-06, 3.2898e-06,\n 4.0866e-06, 3.6385e-06, 3.7895e-06, 3.3973e-06, 4.0610e-06, 3.7689e-06,\n 3.7633e-06, 4.0609e-06, 3.1039e-06, 4.7118e-06, 3.2976e-06, 3.4857e-06,\n 5.9633e-06, 5.0776e-06, 3.4525e-06, 3.6522e-06, 3.2820e-06, 3.6245e-06,\n 3.1844e-06, 5.0350e-06, 3.5139e-06, 4.4792e-06, 4.5443e-06, 3.9182e-06,\n 4.6434e-06, 4.7098e-06, 3.0247e-06, 3.1523e-06, 2.6017e-06, 3.7972e-06,\n 3.8119e-06, 2.3663e-06, 3.6287e-06, 3.1200e-06, 2.3439e-06, 4.1011e-06,\n 2.2178e-06, 3.0065e-06, 5.0115e-06, 9.3372e-06, 5.0056e-06, 3.9786e-06,\n 4.2019e-06, 3.7689e-06, 3.9924e-06, 8.4395e-07, 3.5119e-06, 3.6546e-06,\n 4.4192e-06, 2.3625e-06, 2.7641e-06, 6.6346e-06, 4.0586e-06, 2.8489e-06,\n 1.0051e-06, 3.6671e-06, 2.8865e-12, 3.5510e-06, 3.5730e-06, 4.2901e-06,\n 4.6220e-06, 3.4943e-06, 2.8669e-06, 3.3855e-06, 3.6225e-06, 3.1213e-06,\n 3.3023e-06, 2.8500e-06, 3.1278e-06, 3.6356e-06, 3.8600e-06, 1.3871e-06,\n 3.7393e-06, 3.4816e-06, 4.5497e-06, 3.6107e-06, 4.4850e-06, 3.2089e-06,\n 3.8062e-06, 2.8107e-06, 3.6095e-06, 3.8473e-06, 3.2919e-06, 4.7702e-06,\n 1.0683e-06, 4.4698e-06, 3.3653e-06, 3.3200e-06, 3.4988e-06, 3.5379e-06,\n 4.0502e-06, 5.0970e-06, 4.5616e-06, 3.9654e-06, 2.4805e-06, 4.0371e-06,\n 3.8595e-06, 4.1476e-06, 4.3695e-06, 3.8625e-06, 3.8420e-06, 2.7288e-06,\n 3.7666e-06, 3.6445e-06, 2.2040e-06, 3.3066e-06, 2.9945e-06, 3.8288e-06,\n 3.6958e-07, 1.8158e-06, 1.3714e-06, 6.4138e-06, 4.4826e-06, 5.6744e-11,\n 4.0041e-06, 3.8054e-06, 3.2787e-06, 3.9334e-06, 4.0441e-06, 2.6277e-06,\n 3.6933e-06, 4.0672e-06, 4.3261e-06, 3.6041e-06, 3.1786e-06, 3.1499e-06,\n 3.6967e-06, 4.1632e-06, 3.9127e-06, 3.7801e-06, 3.2559e-06, 3.9173e-06,\n 3.8058e-06, 3.5988e-06, 3.4503e-06, 3.3904e-06, 3.3204e-06, 3.9979e-06,\n 3.8568e-06, 4.0601e-06, 5.8168e-06, 3.0646e-06, 5.8834e-06, 4.1342e-06,\n 6.7114e-07, 3.5196e-06, 4.4227e-06, 3.6502e-06, 4.8855e-06, 3.3491e-06,\n 3.3281e-06, 2.4497e-06, 4.2556e-06, 3.8485e-06, 4.5825e-06, 2.9563e-06,\n 4.0379e-06, 3.2217e-06, 3.5495e-06, 4.8636e-06, 2.8345e-06, 4.4194e-06,\n 3.4899e-06, 1.6918e-06, 3.5393e-06, 3.4915e-06, 5.4734e-06, 4.8742e-06,\n 4.6284e-06, 3.1513e-06, 3.8113e-06, 3.8029e-06, 4.0282e-06, 3.2500e-06,\n 3.5321e-06, 3.3751e-06, 2.8799e-06, 3.7365e-06, 3.2367e-06, 5.1922e-06,\n 4.1393e-06, 6.5090e-13, 3.2230e-06, 3.8625e-06, 3.7991e-06, 4.1853e-06,\n 4.5740e-06, 4.2409e-06, 2.9128e-06, 2.8774e-06, 3.8253e-06, 3.8639e-06,\n 4.0935e-06, 3.3801e-06, 4.3166e-06, 2.2115e-06, 3.4205e-06, 2.7156e-06,\n 3.2625e-06, 3.8261e-06, 2.6208e-06, 5.7527e-06, 4.7129e-06, 4.0080e-06,\n 3.4009e-06, 3.1728e-06, 1.2095e-06, 3.3494e-06, 3.4443e-06, 4.3282e-06,\n 6.8510e-13, 2.8521e-06, 3.5556e-06, 3.4282e-06, 1.2066e-06, 3.4478e-06,\n 2.8841e-06, 3.9809e-06, 3.7985e-06, 3.6283e-06, 3.9386e-06, 4.1067e-06,\n 4.3665e-06, 2.2025e-06, 4.0797e-06, 4.7096e-06, 1.4944e-06, 4.1272e-06,\n 3.4471e-06, 2.3642e-06, 3.4425e-06, 4.6667e-06, 3.2591e-06, 4.4902e-06,\n 3.7173e-06, 3.6904e-06, 3.0417e-06, 5.5325e-06, 3.4769e-06, 3.2607e-06,\n 3.2664e-06, 3.5869e-06, 3.1374e-06, 2.7093e-06, 3.8358e-06, 3.8861e-06,\n 3.3433e-06, 4.2462e-06, 5.1450e-06, 3.2741e-06, 1.5045e-12, 3.9635e-06,\n 2.4742e-06, 2.8666e-06, 4.3104e-06, 3.7689e-06, 3.7177e-06, 3.4883e-06,\n 2.7755e-06, 6.0420e-06, 4.3278e-06, 3.4484e-06, 3.0903e-06, 3.3730e-06,\n 3.6265e-06, 4.0318e-06, 3.5286e-06, 3.3516e-06, 3.7293e-06, 2.1359e-06,\n 4.2998e-06, 3.5703e-06, 3.6291e-06, 5.9452e-11, 4.3234e-06, 3.9559e-06,\n 3.4059e-06, 3.5809e-06, 9.7306e-07, 2.6979e-06, 4.0848e-06, 3.6678e-06,\n 3.7712e-06, 1.6748e-06, 3.6657e-06, 2.6201e-06, 3.7964e-06, 4.5552e-06,\n 4.3411e-06, 3.0684e-06, 4.1289e-06, 3.4665e-06, 1.7074e-12, 3.6090e-06,\n 3.7075e-06, 4.1015e-06, 6.1771e-06, 2.9632e-06, 4.5055e-06, 3.6099e-06,\n 1.6231e-14, 1.1013e-06, 3.2506e-06, 3.5525e-06, 4.4371e-06, 5.3417e-06,\n 4.8583e-06, 3.1620e-06, 4.0548e-06, 3.2398e-06, 3.6087e-06, 3.6534e-06,\n 2.7824e-06, 3.1998e-06, 2.0931e-06, 4.2179e-06, 4.7180e-06, 4.1158e-06,\n 3.4623e-06, 2.8921e-06, 1.9868e-06, 3.5942e-06, 3.0350e-06, 4.2797e-06,\n 5.1696e-06, 3.9319e-06, 5.0863e-06, 3.6903e-06, 3.4856e-06, 3.7452e-06,\n 3.4788e-06, 4.9859e-12, 4.8905e-06, 3.6646e-06, 4.3124e-06, 3.3744e-06,\n 3.6132e-06, 4.8755e-06, 6.9300e-12, 3.9351e-06, 2.6307e-06, 4.3804e-06,\n 3.3856e-06, 3.7289e-06, 3.8054e-06, 3.9658e-06, 2.8508e-06, 4.1267e-06,\n 3.6450e-06, 3.1337e-06, 1.1490e-10, 3.3927e-06, 3.8463e-06, 4.0298e-06,\n 3.0704e-06, 2.0822e-06, 3.6213e-06, 3.7131e-06, 3.3622e-06, 3.2263e-06,\n 1.5872e-10, 2.7554e-06, 2.9957e-06, 3.6018e-06, 3.4711e-06, 3.0420e-06,\n 4.0064e-06, 4.6071e-06, 9.5445e-07, 3.4827e-06, 3.5225e-06, 5.5425e-06,\n 4.2019e-06, 1.0587e-06, 1.3661e-10, 3.9553e-06, 3.4290e-06, 3.4451e-06,\n 4.0415e-06, 3.9466e-06, 2.8423e-06, 4.3845e-06, 3.4664e-06, 4.4643e-06,\n 3.5628e-06, 3.5242e-06, 4.0173e-06, 3.5538e-06, 3.8752e-06, 3.6377e-06,\n 3.6718e-06, 4.5722e-06, 3.5968e-06, 2.1036e-06, 3.6467e-06, 3.2005e-06,\n 2.6161e-06, 3.8978e-06, 3.1531e-06, 3.3507e-06, 3.6072e-06, 3.4833e-06,\n 5.7233e-06, 5.3430e-06, 4.0027e-06, 3.2110e-06, 3.5276e-06, 6.0404e-06,\n 4.1281e-06, 3.3626e-06, 3.9309e-06, 3.7837e-06, 3.6523e-06, 3.5227e-06,\n 3.6172e-06, 3.3177e-06, 3.3622e-06, 4.7983e-06, 3.3686e-06, 3.8222e-06,\n 3.8324e-06, 4.6631e-06, 3.5651e-06, 5.3828e-06, 4.0946e-06, 3.4526e-06,\n 2.5891e-06, 3.6940e-06, 3.7388e-06, 3.8310e-06, 3.7455e-06, 4.1971e-06,\n 1.9248e-06, 3.5755e-06, 3.1032e-06, 2.2570e-06, 2.7540e-06, 2.4749e-06,\n 3.7738e-06, 3.3697e-06, 3.7034e-06, 3.7379e-06, 3.6088e-06, 4.4023e-06,\n 3.4161e-06, 1.9873e-06, 3.2489e-06, 5.2316e-06, 6.5069e-06, 3.7195e-06,\n 4.2138e-06, 2.4879e-06, 3.4848e-06, 3.4720e-06, 4.1089e-06, 3.6046e-06,\n 3.2674e-06, 2.6170e-06, 4.1833e-06, 3.8331e-06, 3.7860e-06, 4.2774e-06,\n 1.3657e-06, 3.3370e-06, 5.1530e-06, 9.6913e-07, 4.2720e-06, 3.3386e-06,\n 3.7174e-06, 3.6187e-06, 4.1972e-06, 3.8035e-06, 3.9962e-06, 3.4466e-06,\n 2.6213e-06, 3.8191e-06, 1.7003e-06, 4.0614e-06, 3.1409e-06, 4.7847e-06,\n 2.5527e-06, 4.0637e-06, 3.2101e-06, 3.3761e-06, 3.3828e-06, 3.9931e-06,\n 4.5001e-06, 3.1321e-06, 3.6071e-06, 3.2806e-06, 2.6580e-06, 3.5454e-06,\n 3.5447e-06, 3.0622e-06, 4.2789e-06, 3.2975e-06, 2.6241e-06, 5.6408e-06,\n 4.6328e-06, 3.3039e-06, 3.3096e-06, 4.0392e-06, 3.6336e-06, 4.0317e-06,\n 3.4531e-06, 3.7864e-06, 3.7978e-06, 3.3378e-06, 1.8557e-06, 3.8389e-06,\n 2.4200e-06, 1.2982e-11, 2.9435e-06, 4.0559e-06, 3.9525e-06, 4.0631e-06,\n 3.9265e-06, 4.9353e-06, 3.1632e-06, 3.1093e-06, 4.7506e-06, 4.3543e-06,\n 4.9341e-06, 3.8663e-06, 3.4888e-06, 4.4649e-06, 3.4722e-06, 3.6133e-06,\n 3.8880e-06, 3.9555e-06, 2.2928e-06, 3.9570e-06, 4.1501e-06, 3.8057e-06,\n 3.1837e-06, 4.0043e-06, 4.2156e-06, 3.9618e-06, 3.0541e-06, 3.3393e-06,\n 3.4075e-06, 2.5344e-06, 3.1465e-06, 3.6014e-06, 4.5935e-06, 3.8446e-06,\n 3.3987e-06, 3.1368e-06, 3.2032e-06, 1.3293e-06, 3.1969e-06, 3.9727e-07,\n 3.1702e-06, 3.6113e-06, 3.5676e-06, 3.0245e-06, 3.5631e-06, 3.7367e-06,\n 3.7163e-06, 4.0112e-06, 3.4691e-06, 3.8028e-06, 5.0662e-06, 3.9719e-06,\n 8.1644e-06, 4.6195e-06, 3.7997e-06, 4.6131e-06, 4.5744e-06, 3.4832e-06,\n 4.8181e-06, 4.0268e-06, 3.5457e-06, 4.6587e-06, 4.1930e-06, 3.6955e-06,\n 2.0149e-06, 4.3793e-06, 3.5667e-06, 4.2894e-06, 4.9014e-06, 3.7352e-06,\n 2.9060e-06, 3.9869e-06, 4.3947e-06, 3.5181e-06, 5.0579e-06, 5.4493e-06,\n 2.9906e-06, 3.4491e-06, 3.7718e-06, 3.3646e-06, 4.3686e-06, 3.5723e-06,\n 3.3607e-06, 2.3284e-06, 4.6521e-06, 5.1713e-06, 4.6400e-06, 4.0426e-06,\n 3.9107e-06, 4.0839e-06, 3.5948e-06, 3.7009e-06, 3.8793e-06, 3.6775e-06,\n 3.0678e-06, 3.1297e-06, 2.8902e-06, 3.4144e-06, 3.4508e-06, 3.7578e-06,\n 7.3983e-06, 4.1640e-06, 4.0789e-06, 3.7763e-06, 4.1818e-06, 5.8280e-06,\n 4.2562e-06, 3.5242e-06, 3.6316e-06, 4.0303e-06, 4.4293e-06, 3.8569e-06,\n 5.1178e-06, 6.6270e-06, 3.6622e-06, 3.4732e-06, 3.3762e-06, 2.2642e-06,\n 3.2855e-06, 3.1908e-06, 9.0680e-07, 3.4284e-06, 3.6160e-06, 4.3826e-06,\n 3.7088e-06, 1.9054e-06, 1.1767e-06, 4.0345e-06, 3.4722e-06, 3.1950e-06,\n 3.8897e-06, 3.4246e-06, 3.9167e-06, 5.5621e-06, 3.9191e-06, 3.8333e-06,\n 3.4901e-06, 4.2492e-06, 3.3405e-06, 3.6831e-06, 4.0799e-06, 3.8223e-06,\n 1.9880e-06, 4.2193e-06, 3.3325e-06, 4.6613e-06, 3.5351e-06, 3.8545e-06],\n device='cuda:0')" + }, + "4": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 2.1564e-06, 2.1110e-06, -2.7713e-06, ..., -5.4352e-07,\n -2.9311e-07, 6.3135e-06],\n [ 8.9507e-06, -8.1639e-06, 2.8635e-06, ..., 1.9788e-05,\n 1.7167e-07, -2.1505e-05],\n [ 5.5091e-06, -6.5758e-06, 2.0536e-05, ..., 3.9009e-06,\n 4.2352e-06, 1.6457e-05],\n ...,\n [-6.9249e-06, 3.6042e-06, 5.4859e-06, ..., -1.0687e-05,\n -1.8745e-06, -2.3892e-07],\n [ 8.2036e-06, 7.6792e-06, 3.5991e-08, ..., -1.3707e-05,\n -2.9498e-07, -5.8306e-06],\n [ 2.6378e-06, 7.0123e-06, -7.0993e-06, ..., -4.8683e-06,\n 7.4787e-06, 2.1983e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.8973e-10, 5.5680e-10, 3.1332e-10, ..., 2.0689e-10, 7.8140e-10,\n 1.1482e-09],\n [1.3707e-09, 1.4893e-09, 5.6545e-10, ..., 4.3483e-10, 1.2574e-09,\n 1.5682e-09],\n [1.2291e-09, 1.8447e-09, 9.3320e-10, ..., 6.2068e-10, 1.4217e-09,\n 2.2520e-09],\n ...,\n [1.1229e-09, 1.2058e-09, 5.9561e-10, ..., 6.7221e-10, 1.2753e-09,\n 1.9167e-09],\n [1.3335e-09, 1.1610e-09, 7.5184e-10, ..., 1.0154e-09, 1.7898e-09,\n 1.8994e-09],\n [1.3271e-09, 1.4691e-09, 8.9503e-10, ..., 5.2081e-10, 1.5999e-09,\n 1.9807e-09]], device='cuda:0')" + }, + "5": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -1.3455e-06, 4.1459e-06, ..., -2.3722e-06,\n 1.0328e-06, 1.4545e-05],\n [-5.6052e-45, -9.6284e-07, -6.2206e-09, ..., -1.6139e-08,\n -1.2786e-05, 9.6101e-08],\n [ 5.6052e-45, 8.2402e-07, -5.6116e-07, ..., 1.4373e-06,\n 5.5635e-07, 3.0782e-06],\n ...,\n [ 5.6052e-45, 3.6645e-05, -1.6689e-06, ..., 1.1938e-06,\n -4.3547e-06, -5.5566e-06],\n [ 5.6052e-45, 9.2744e-07, -1.3569e-07, ..., -1.0330e-05,\n 1.5774e-06, 2.1170e-05],\n [ 5.6052e-45, 2.7846e-05, -5.0362e-06, ..., -3.2622e-06,\n 5.5648e-06, -2.9343e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.4185e-14, 4.1397e-10, 7.5635e-10, ..., 7.7460e-10, 3.7226e-09,\n 1.7136e-09],\n [1.0581e-13, 2.4440e-10, 5.8070e-11, ..., 1.0943e-10, 1.4941e-09,\n 1.1707e-09],\n [4.5916e-15, 5.7985e-10, 1.4413e-09, ..., 8.0015e-10, 3.1668e-10,\n 7.8077e-10],\n ...,\n [1.1706e-14, 1.2668e-09, 5.2047e-09, ..., 4.4953e-10, 2.8714e-10,\n 9.7217e-11],\n [5.0504e-14, 8.6017e-10, 2.8435e-11, ..., 4.1661e-09, 1.2956e-10,\n 2.8459e-09],\n [1.7185e-13, 1.9151e-09, 8.4732e-10, ..., 1.6599e-09, 4.4247e-10,\n 3.2051e-10]], device='cuda:0')" + }, + "6": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 0.0003, -0.0003, -0.0002, ..., 0.0001, 0.0004, 0.0005],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.1885e-06, 2.2977e-06, 2.4368e-06, ..., 3.3536e-06, 2.9280e-06,\n 3.1204e-06], device='cuda:0')" + }, + "7": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-3.5692e-06, -1.0190e-07, -2.8779e-06, ..., 1.0793e-05,\n 9.1000e-07, 2.4148e-06],\n [ 1.4791e-05, 3.5880e-06, 3.3272e-06, ..., 1.3399e-05,\n -6.7962e-07, 1.0785e-05],\n [-1.1875e-05, -8.6956e-07, -6.1965e-06, ..., 5.0239e-06,\n 1.2863e-06, -1.7836e-06],\n ...,\n [-2.5900e-06, -5.8003e-06, -5.4922e-06, ..., -4.4815e-06,\n -1.7362e-06, -5.9966e-06],\n [ 9.6900e-07, -1.4988e-06, 5.8693e-06, ..., 7.3401e-06,\n 2.3731e-06, 2.0996e-06],\n [ 6.4682e-06, 8.1482e-07, 5.0291e-06, ..., -9.6564e-06,\n 1.0795e-06, 3.1887e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.3373e-10, 9.4496e-11, 2.6558e-10, ..., 4.0417e-10, 1.7596e-10,\n 3.7145e-10],\n [5.3886e-10, 1.6860e-10, 5.2380e-10, ..., 5.9315e-10, 2.4145e-10,\n 5.9879e-10],\n [7.6506e-10, 3.1340e-10, 9.5717e-10, ..., 5.1364e-10, 3.3133e-10,\n 6.2515e-10],\n ...,\n [5.8036e-10, 1.8830e-10, 6.8538e-10, ..., 6.0901e-10, 3.1071e-10,\n 8.5639e-10],\n [5.2115e-10, 2.2155e-10, 6.7218e-10, ..., 4.8816e-10, 2.2892e-10,\n 7.4624e-10],\n [5.5805e-10, 1.1736e-10, 8.6656e-10, ..., 6.3615e-10, 4.8403e-10,\n 5.4347e-10]], device='cuda:0')" + }, + "8": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -1.3206e-04, -8.2335e-06, ..., 2.3357e-07,\n -4.8956e-07, -6.4666e-06],\n [ 5.6052e-45, 8.3937e-05, 1.3233e-06, ..., -1.3466e-06,\n 7.1205e-06, 5.0914e-07],\n [-5.6052e-45, 9.3344e-05, -3.3259e-06, ..., 2.1313e-05,\n 2.8684e-05, 4.0527e-06],\n ...,\n [ 5.6052e-45, 1.4280e-05, 1.9399e-08, ..., 3.1737e-06,\n 1.6201e-05, 2.6687e-06],\n [ 5.6052e-45, -3.0657e-06, -1.1953e-05, ..., -3.9274e-06,\n 8.1458e-06, 4.5741e-05],\n [ 5.6052e-45, -6.2335e-06, -1.3931e-05, ..., -5.4994e-05,\n 4.5396e-06, 2.0338e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.2328e-15, 9.6822e-08, 3.7055e-10, ..., 2.4630e-09, 7.6038e-10,\n 2.7422e-09],\n [3.0801e-13, 1.7028e-07, 5.2504e-09, ..., 2.6624e-09, 2.2979e-08,\n 9.6376e-09],\n [6.6443e-14, 7.1239e-08, 2.0352e-09, ..., 5.1676e-08, 2.0226e-08,\n 2.0540e-09],\n ...,\n [3.0893e-16, 3.4665e-08, 9.5641e-10, ..., 5.3388e-09, 1.2148e-08,\n 4.0536e-09],\n [4.4355e-14, 4.8438e-09, 7.8148e-09, ..., 4.0980e-09, 1.7204e-08,\n 8.0990e-08],\n [4.9034e-14, 8.8642e-09, 2.2698e-08, ..., 9.5100e-08, 6.3403e-08,\n 1.3910e-08]], device='cuda:0')" + }, + "9": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-2.8339e-05, 9.8823e-05, 5.5747e-03, ..., 1.4112e-03,\n 2.3745e-03, -1.4952e-03], device='cuda:0')", + "exp_avg_sq": "tensor([1.5341e-04, 9.5247e-05, 1.1766e-04, ..., 9.5489e-05, 1.1487e-04,\n 1.4182e-04], device='cuda:0')" + }, + "10": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-8.2245e-07, 1.1188e-05, 1.1900e-05, ..., 3.1138e-05,\n -1.2341e-05, -3.9764e-06],\n [-1.2487e-07, 4.0061e-05, 5.9625e-06, ..., -2.2870e-05,\n 3.3781e-06, 5.1458e-06],\n [-3.5879e-06, 7.0330e-07, -7.4589e-06, ..., 6.9924e-06,\n 2.3694e-06, -1.0935e-05],\n ...,\n [ 2.4829e-05, 1.0584e-05, 1.1950e-05, ..., 4.0562e-05,\n -4.2175e-06, 1.8129e-05],\n [-2.6313e-05, 8.7631e-06, -2.5497e-05, ..., 4.3825e-05,\n -8.7733e-06, -3.4352e-05],\n [-4.6378e-06, 1.9407e-05, 1.7410e-05, ..., -3.7780e-06,\n 8.0232e-07, -4.7834e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.2283e-09, 2.2702e-09, 5.1447e-09, ..., 1.6926e-08, 1.6406e-09,\n 2.9476e-09],\n [9.8448e-09, 5.3086e-09, 1.1128e-08, ..., 1.2418e-08, 3.5559e-09,\n 9.2991e-09],\n [3.4954e-09, 5.2968e-09, 8.5104e-09, ..., 1.3407e-08, 3.7945e-09,\n 6.8898e-09],\n ...,\n [4.6170e-09, 3.6578e-09, 9.7530e-09, ..., 2.3057e-08, 8.5501e-09,\n 7.5672e-09],\n [4.2037e-09, 2.8470e-09, 1.2323e-08, ..., 1.1137e-08, 8.1705e-09,\n 7.3299e-09],\n [3.9305e-09, 4.2795e-09, 1.0935e-08, ..., 1.3401e-08, 3.0956e-09,\n 1.4413e-08]], device='cuda:0')" + }, + "11": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -1.4186e-06, 2.3587e-08, ..., -5.5311e-06,\n 1.4654e-06, -3.1021e-07],\n [ 5.6052e-45, 1.1599e-07, 2.4306e-07, ..., 6.8919e-06,\n -2.6733e-06, 1.8757e-07],\n [-5.6052e-45, -3.5290e-07, 4.5880e-08, ..., -1.1201e-06,\n -2.1904e-06, 1.7164e-06],\n ...,\n [-5.6052e-45, -3.2420e-06, -1.8024e-07, ..., -9.9164e-07,\n -4.9331e-06, 7.1539e-06],\n [ 5.6052e-45, 9.4678e-06, 4.3918e-07, ..., 1.8045e-07,\n 5.6886e-06, -8.1956e-09],\n [ 5.6052e-45, 5.6120e-06, -9.5804e-08, ..., -3.4096e-06,\n 8.0547e-07, 1.7728e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.7313e-17, 4.0681e-10, 2.5007e-11, ..., 5.2909e-11, 2.5398e-10,\n 1.4736e-09],\n [7.2126e-14, 1.9269e-10, 3.2291e-12, ..., 2.0447e-10, 1.0571e-10,\n 7.9377e-11],\n [1.9785e-14, 1.2852e-10, 1.2873e-10, ..., 1.9173e-10, 3.7975e-10,\n 1.0731e-10],\n ...,\n [3.5495e-14, 1.3794e-10, 1.8497e-10, ..., 6.5961e-10, 4.8794e-10,\n 6.9354e-10],\n [3.9491e-15, 3.1486e-09, 2.9115e-10, ..., 1.2340e-10, 4.2062e-10,\n 6.3856e-11],\n [1.7980e-16, 1.8070e-10, 1.3496e-11, ..., 5.0528e-10, 1.8499e-10,\n 1.5952e-10]], device='cuda:0')" + }, + "12": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.2675e-04, 3.1104e-05, -1.3252e-04, ..., 3.4319e-04,\n 2.1374e-04, -2.5188e-05], device='cuda:0')", + "exp_avg_sq": "tensor([1.1714e-06, 1.0478e-06, 1.4390e-06, ..., 2.1524e-06, 1.8328e-06,\n 1.0128e-06], device='cuda:0')" + }, + "13": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 1.8191e-07, 4.7228e-07, 9.6992e-07, ..., -1.6600e-06,\n -1.4939e-06, 4.3950e-07],\n [ 1.4369e-06, -2.2105e-07, 2.2115e-06, ..., -2.1756e-06,\n 1.0206e-06, 3.0971e-06],\n [ 7.2600e-07, 2.7575e-07, 4.7995e-07, ..., 2.0604e-06,\n -5.9336e-07, 6.7236e-06],\n ...,\n [-3.1716e-06, 6.3263e-07, -1.9599e-06, ..., 5.4397e-07,\n -4.8225e-08, -5.5020e-06],\n [-5.2778e-07, -3.3120e-07, -2.4423e-06, ..., 2.0885e-06,\n -4.6205e-07, 8.5490e-07],\n [ 5.3776e-07, -1.5771e-06, -1.1625e-08, ..., 2.0213e-07,\n 1.5635e-06, -3.7703e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7696e-11, 3.4328e-11, 3.0745e-11, ..., 4.1601e-11, 3.1516e-11,\n 3.4158e-11],\n [7.9940e-11, 8.8110e-11, 1.0330e-10, ..., 6.5966e-11, 6.7190e-11,\n 5.9660e-11],\n [1.1700e-10, 6.3763e-11, 7.3777e-11, ..., 5.0805e-11, 5.9505e-11,\n 5.7774e-11],\n ...,\n [5.1363e-11, 7.2067e-11, 7.7590e-11, ..., 5.9412e-11, 6.4776e-11,\n 6.5001e-11],\n [3.2232e-11, 6.8081e-11, 7.4525e-11, ..., 5.2406e-11, 7.4076e-11,\n 8.0702e-11],\n [5.3243e-11, 7.4483e-11, 8.3699e-11, ..., 8.0153e-11, 8.6263e-11,\n 8.6295e-11]], device='cuda:0')" + }, + "14": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 5.6052e-45, 4.8239e-07, -3.8964e-08, ..., -2.3117e-08,\n -3.7151e-06, 1.3280e-07],\n [-5.6052e-45, 3.8894e-10, 1.7976e-23, ..., 1.5198e-09,\n -8.1069e-09, 1.2898e-08],\n [ 5.6052e-45, 8.1403e-07, 4.5923e-07, ..., -4.0361e-07,\n -1.0791e-05, 1.4508e-08],\n ...,\n [ 5.6052e-45, -2.6558e-07, 2.4925e-05, ..., -2.8603e-08,\n -8.9921e-07, 5.1096e-08],\n [-5.6052e-45, 1.4914e-09, 1.2312e-06, ..., 7.2885e-07,\n -3.0345e-07, -1.6256e-06],\n [ 5.6052e-45, 3.8692e-11, 1.5056e-06, ..., 1.8772e-09,\n 5.5810e-07, 7.6283e-09]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7950e-14, 1.1749e-09, 6.6199e-11, ..., 4.5619e-13, 3.1390e-10,\n 2.2684e-12],\n [3.4003e-14, 5.7193e-12, 2.2983e-14, ..., 8.8886e-12, 2.3685e-11,\n 1.3851e-13],\n [8.7992e-15, 1.3640e-11, 8.7821e-11, ..., 2.5794e-10, 3.5544e-10,\n 2.8665e-12],\n ...,\n [8.2190e-15, 3.9819e-10, 8.5382e-09, ..., 3.1343e-11, 2.5647e-10,\n 1.3302e-11],\n [3.2477e-16, 2.9795e-11, 2.6634e-09, ..., 4.4324e-11, 1.5188e-10,\n 2.5025e-10],\n [1.4855e-14, 5.9903e-13, 1.1950e-12, ..., 6.5066e-13, 1.2811e-11,\n 3.4487e-14]], device='cuda:0')" + }, + "15": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.4609e-04, 1.2732e-04, -2.8415e-05, ..., 1.6301e-04,\n 3.7829e-05, 4.5382e-05], device='cuda:0')", + "exp_avg_sq": "tensor([2.2992e-06, 1.1474e-06, 9.5888e-07, ..., 1.4877e-06, 1.0660e-06,\n 2.2528e-08], device='cuda:0')" + }, + "16": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-3.0304e-07, -5.4351e-08, 3.9574e-07, ..., -5.9697e-07,\n -1.0343e-06, 1.8252e-08],\n [ 1.3447e-07, 7.9635e-08, -4.7518e-07, ..., -5.3258e-07,\n -3.4093e-07, 3.0065e-08],\n [-2.2557e-06, -1.4482e-07, -3.4231e-07, ..., -2.6369e-07,\n 6.4383e-07, 8.3723e-08],\n ...,\n [ 2.0712e-06, 3.2056e-07, 5.2321e-07, ..., 4.7367e-07,\n 1.9777e-07, -2.3645e-09],\n [-5.9990e-09, -7.5420e-07, -2.0125e-06, ..., 1.5081e-07,\n 9.7747e-07, 2.3246e-08],\n [ 1.0174e-07, -1.0260e-07, -9.5623e-07, ..., -2.1106e-07,\n -1.0870e-06, 1.4025e-09]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.7898e-12, 2.3753e-12, 9.8447e-12, ..., 4.4663e-11, 9.4254e-11,\n 3.6580e-13],\n [3.4001e-11, 9.6541e-12, 2.2346e-11, ..., 2.4549e-11, 3.3156e-11,\n 3.5855e-12],\n [1.5506e-10, 1.1970e-11, 1.1692e-11, ..., 3.1759e-11, 2.8773e-11,\n 2.5461e-12],\n ...,\n [2.9909e-10, 6.1528e-12, 1.1210e-11, ..., 4.1374e-11, 2.2508e-11,\n 9.9777e-13],\n [3.9164e-11, 9.4399e-12, 1.9868e-11, ..., 8.7262e-12, 2.3286e-11,\n 1.1627e-12],\n [3.0704e-11, 1.6607e-11, 2.4003e-11, ..., 1.5006e-11, 8.0248e-11,\n 8.4095e-13]], device='cuda:0')" + }, + "17": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -9.0947e-07, -4.0356e-08, ..., 2.5269e-06,\n 4.6838e-06, 3.4731e-06],\n [ 5.6052e-45, -5.9409e-06, 7.4716e-10, ..., 1.3724e-07,\n 5.6144e-09, 5.9717e-07],\n [ 5.6052e-45, -4.5733e-06, 1.8039e-06, ..., -4.9665e-07,\n 5.6977e-06, -1.1490e-05],\n ...,\n [ 5.6052e-45, -9.1596e-27, -5.6052e-45, ..., 5.1265e-36,\n 3.2277e-24, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., 5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [-5.6052e-45, -4.6912e-08, 6.7068e-09, ..., -2.8540e-06,\n -5.0570e-06, 3.8340e-07]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.0841e-16, 1.4391e-10, 9.4633e-12, ..., 2.9727e-10, 6.3636e-10,\n 2.1955e-10],\n [1.7887e-14, 4.4066e-09, 1.8786e-10, ..., 5.6081e-10, 8.5112e-12,\n 9.1537e-10],\n [6.7314e-16, 5.1813e-10, 1.4932e-11, ..., 3.9659e-10, 4.2088e-10,\n 2.8360e-09],\n ...,\n [2.2544e-14, 2.5940e-14, 9.1610e-16, ..., 1.2624e-14, 1.0803e-12,\n 6.9142e-16],\n [1.8691e-17, 7.4365e-17, 4.9205e-15, ..., 2.9617e-14, 4.0987e-16,\n 6.5399e-15],\n [3.2080e-16, 1.5099e-10, 1.9918e-10, ..., 2.9565e-10, 1.3615e-10,\n 4.6666e-10]], device='cuda:0')" + }, + "18": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.3128e-04, 9.8099e-06, -1.9001e-04, ..., 8.6198e-23,\n 5.6052e-45, 1.8161e-04], device='cuda:0')", + "exp_avg_sq": "tensor([1.6450e-06, 1.7367e-06, 1.3319e-06, ..., 2.4191e-10, 1.6470e-10,\n 1.1251e-06], device='cuda:0')" + }, + "19": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 1.3961e-06, -2.8630e-07, -7.9430e-08, ..., 6.6697e-26,\n 5.6052e-45, 7.1475e-07],\n [ 2.5787e-06, -3.5734e-07, -3.8106e-07, ..., -2.3084e-26,\n -5.6052e-45, -3.1252e-07],\n [-1.0413e-06, -3.4634e-07, 8.1080e-07, ..., -2.8271e-25,\n -5.6052e-45, 1.7072e-06],\n ...,\n [ 3.0546e-06, -1.5643e-06, -7.8837e-07, ..., -7.8925e-27,\n -5.6052e-45, -8.9867e-07],\n [-1.7204e-06, 1.3981e-06, 1.2514e-06, ..., 5.2392e-26,\n 5.6052e-45, 7.2250e-07],\n [ 2.6615e-06, -1.6223e-06, -2.5975e-07, ..., -2.1016e-25,\n 5.6052e-45, -1.0558e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.2857e-11, 3.1021e-12, 1.8220e-12, ..., 9.7375e-16, 7.9552e-16,\n 1.3802e-11],\n [1.7431e-11, 9.0562e-12, 3.2483e-12, ..., 2.7679e-15, 5.3222e-16,\n 1.4658e-11],\n [2.4431e-11, 2.5905e-12, 9.0357e-12, ..., 1.8173e-15, 1.8051e-15,\n 2.3975e-11],\n ...,\n [2.3319e-11, 4.7673e-11, 7.4900e-12, ..., 2.9671e-15, 1.1573e-15,\n 2.2839e-11],\n [3.1652e-11, 1.2769e-10, 1.1004e-11, ..., 4.5027e-15, 1.8118e-15,\n 2.5948e-11],\n [2.6055e-11, 1.9282e-10, 1.5953e-11, ..., 5.0796e-15, 1.9355e-15,\n 1.9207e-11]], device='cuda:0')" + }, + "20": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 5.6052e-45, 1.0220e-06, 8.9670e-09, ..., -3.0754e-06,\n 6.3202e-07, 7.5757e-06],\n [ 5.6052e-45, -2.8155e-14, 1.8469e-08, ..., 7.6832e-09,\n -1.6724e-07, 1.6032e-13],\n [ 5.6052e-45, -3.6561e-07, 2.7198e-07, ..., 1.4162e-06,\n -4.1139e-06, -6.4116e-07],\n ...,\n [ 5.6052e-45, 2.0934e-07, 4.4067e-11, ..., 1.5472e-08,\n 7.3951e-06, -7.4403e-18],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n -5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, -3.7859e-06, 4.9818e-08, ..., -1.2798e-06,\n 7.5178e-07, 1.5623e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.2333e-15, 8.6360e-10, 6.2422e-11, ..., 7.1072e-10, 6.2121e-10,\n 8.2284e-11],\n [1.0320e-14, 8.3340e-14, 2.6022e-12, ..., 9.3741e-12, 9.6086e-12,\n 1.4097e-12],\n [2.1116e-15, 4.2102e-11, 8.4602e-10, ..., 1.5732e-10, 4.1234e-10,\n 1.1506e-10],\n ...,\n [2.7216e-18, 1.0772e-10, 1.9961e-13, ..., 7.1832e-12, 2.2416e-09,\n 1.2287e-12],\n [3.7053e-20, 4.1303e-20, 8.1860e-20, ..., 4.4072e-20, 4.7327e-20,\n 2.4195e-20],\n [1.3041e-15, 5.2585e-10, 2.6776e-11, ..., 7.0996e-11, 9.0115e-10,\n 3.4233e-10]], device='cuda:0')" + }, + "21": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.1469e-04, 5.6894e-06, 3.3939e-04, ..., -1.0625e-04,\n 5.6052e-45, 5.6388e-05], device='cuda:0')", + "exp_avg_sq": "tensor([1.4811e-06, 5.7866e-08, 2.3092e-06, ..., 1.4426e-06, 7.6455e-15,\n 1.8840e-06], device='cuda:0')" + }, + "22": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 6.3568e-07, 1.4365e-08, -1.8465e-07, ..., 1.8698e-07,\n 5.6052e-45, -2.1696e-08],\n [ 5.7098e-07, 2.0026e-08, 5.4812e-07, ..., 3.3728e-07,\n 5.6052e-45, -2.1714e-07],\n [ 1.6792e-06, -3.3540e-08, -9.4233e-08, ..., -8.2710e-08,\n -5.6052e-45, -1.3065e-07],\n ...,\n [-1.7577e-07, 3.9361e-08, 7.9439e-07, ..., 1.6012e-08,\n -5.6052e-45, 4.3631e-07],\n [ 6.5553e-08, 1.1914e-08, -1.1197e-06, ..., 6.0106e-07,\n 5.6052e-45, 2.5737e-08],\n [ 9.6412e-08, 2.1353e-08, -6.0607e-07, ..., -3.3065e-07,\n 5.6052e-45, -8.0405e-08]], device='cuda:0')", + "exp_avg_sq": "tensor([[7.5456e-12, 3.3755e-13, 6.7725e-12, ..., 3.5124e-12, 2.7540e-18,\n 6.5413e-12],\n [1.9328e-11, 4.1660e-13, 1.4660e-11, ..., 6.2759e-12, 5.5753e-18,\n 1.2221e-11],\n [1.6992e-11, 6.3353e-13, 2.0573e-11, ..., 8.3629e-12, 3.2370e-18,\n 1.3528e-11],\n ...,\n [1.2315e-11, 5.1134e-13, 1.6669e-11, ..., 1.2418e-11, 1.3217e-18,\n 1.0922e-11],\n [1.7645e-11, 2.7502e-13, 1.4333e-11, ..., 1.0458e-11, 2.3179e-18,\n 1.1746e-11],\n [8.5691e-12, 4.3895e-13, 1.3526e-11, ..., 7.3210e-12, 8.6439e-19,\n 1.8384e-11]], device='cuda:0')" + }, + "23": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n [ 5.6052e-45, -5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 4.4666e-06, 1.7982e-05, ..., -2.3595e-07,\n 1.2933e-07, 5.0961e-08],\n ...,\n [-5.6052e-45, 5.6052e-45, 5.6052e-45, ..., 0.0000e+00,\n 5.6052e-45, 5.6052e-45],\n [ 5.6052e-45, 5.6052e-45, -5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-5.6052e-45, 5.6052e-45, -3.3687e-42, ..., -3.7320e-17,\n -1.2817e-16, -7.1542e-36]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.5317e-16, 3.2694e-17, 6.5881e-19, ..., 2.0544e-17, 5.1907e-17,\n 7.9858e-16],\n [2.1099e-15, 2.5752e-15, 3.9608e-15, ..., 4.9974e-16, 7.0730e-16,\n 1.9480e-15],\n [5.4754e-15, 1.9817e-09, 2.4904e-08, ..., 1.9512e-11, 9.7060e-11,\n 1.8668e-11],\n ...,\n [3.7381e-22, 1.7181e-19, 1.2100e-18, ..., 0.0000e+00, 2.2092e-22,\n 1.3813e-21],\n [7.4431e-16, 1.4084e-15, 3.7065e-15, ..., 3.4162e-16, 2.5309e-14,\n 3.5991e-15],\n [2.1863e-15, 2.0838e-15, 1.4412e-12, ..., 1.0596e-12, 2.7717e-13,\n 1.9066e-13]], device='cuda:0')" + }, + "24": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 5.6052e-45, 5.6052e-45, 1.4266e-04, ..., 5.6052e-45,\n 7.2146e-35, -1.4020e-08], device='cuda:0')", + "exp_avg_sq": "tensor([2.7580e-12, 3.2009e-12, 1.1463e-06, ..., 1.4932e-15, 4.9473e-10,\n 1.0698e-09], device='cuda:0')" + }, + "25": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, 5.6052e-45, 1.1254e-06, ..., -5.6052e-45,\n -3.5802e-37, -2.3454e-12],\n [-5.6052e-45, -5.6052e-45, -6.3912e-07, ..., 5.6052e-45,\n -1.2886e-37, 3.1508e-11],\n [-5.6052e-45, 5.6052e-45, 9.3684e-07, ..., -5.6052e-45,\n -5.8173e-37, 2.0671e-11],\n ...,\n [ 5.6052e-45, 5.6052e-45, -1.6730e-06, ..., 5.6052e-45,\n 1.0742e-37, -1.9836e-12],\n [ 5.6052e-45, -5.6052e-45, 3.1223e-07, ..., -5.6052e-45,\n -7.5154e-37, 5.8675e-11],\n [-5.6052e-45, -5.6052e-45, -2.4596e-08, ..., -5.6052e-45,\n -1.6928e-37, -2.5721e-11]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.1470e-17, 1.5963e-15, 6.4598e-12, ..., 1.1876e-20, 1.2302e-14,\n 6.2303e-15],\n [9.4457e-17, 1.2862e-15, 5.2246e-12, ..., 4.8617e-21, 2.7465e-14,\n 5.5265e-15],\n [1.3762e-16, 2.0328e-15, 8.0192e-12, ..., 1.3182e-20, 3.2427e-14,\n 1.4529e-14],\n ...,\n [1.1687e-16, 8.9843e-15, 1.6069e-11, ..., 4.4044e-21, 7.1404e-15,\n 1.3246e-14],\n [1.2155e-16, 8.0910e-15, 3.5192e-12, ..., 3.2776e-20, 1.3228e-14,\n 1.4847e-14],\n [7.7538e-17, 2.4044e-15, 6.3717e-12, ..., 3.1400e-21, 3.0165e-14,\n 2.1529e-14]], device='cuda:0')" + }, + "26": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-2.8486e-10], device='cuda:0')", + "exp_avg_sq": "tensor([4.5417e-09], device='cuda:0')" + }, + "27": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.0108e-12, 1.0846e-12, -2.0954e-12], device='cuda:0')", + "exp_avg_sq": "tensor([7.1825e-12, 2.4657e-12, 5.6552e-12], device='cuda:0')" + }, + "28": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 7.1674e-12, -1.9442e-19, 1.6408e-11, -7.4759e-17, -9.4299e-15,\n -1.0895e-15, -5.4663e-14, -2.6556e-14], device='cuda:0')", + "exp_avg_sq": "tensor([6.3336e-10, 1.1706e-10, 9.1163e-10, 3.0898e-11, 2.1725e-10, 6.0624e-11,\n 3.4072e-10, 1.6131e-10], device='cuda:0')" + }, + "30": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, 8.4930e-17, -1.7500e-17, ..., 4.8661e-17,\n 3.5331e-16, 2.6321e-17],\n [ 5.6052e-45, 3.1862e-16, 6.4295e-18, ..., 6.0300e-16,\n -2.2595e-15, 2.2565e-17],\n [-5.6052e-45, -1.7198e-16, 3.4621e-17, ..., 1.0302e-15,\n -1.0693e-15, 7.0870e-17],\n ...,\n [-5.6052e-45, -1.8694e-16, -5.0062e-17, ..., -3.0141e-15,\n -1.1975e-15, -8.0190e-17],\n [ 5.6052e-45, 5.1473e-16, -1.7195e-17, ..., -1.3449e-15,\n -7.5109e-16, -1.8193e-17],\n [-5.6052e-45, -2.9939e-16, -1.2758e-17, ..., -6.0056e-16,\n 4.9297e-15, -1.3543e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[4.3324e-16, 6.4206e-16, 1.5972e-15, ..., 6.4831e-18, 6.1302e-17,\n 7.5516e-16],\n [4.6006e-15, 7.0234e-15, 8.0884e-15, ..., 3.4703e-17, 4.6469e-16,\n 7.2859e-15],\n [3.1305e-16, 6.8277e-16, 9.8793e-16, ..., 6.9602e-18, 5.3122e-17,\n 2.8749e-16],\n ...,\n [2.1120e-15, 2.6323e-15, 2.7688e-15, ..., 9.4948e-17, 4.8639e-16,\n 2.5169e-15],\n [9.9714e-17, 4.0233e-16, 2.0567e-16, ..., 1.0647e-19, 2.6606e-17,\n 1.5411e-16],\n [2.1687e-17, 1.2088e-16, 3.9588e-17, ..., 1.5324e-19, 1.9482e-18,\n 1.8909e-17]], device='cuda:0')" + }, + "31": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.2762e-12, 3.1647e-13, 2.2323e-12, -5.9161e-13, -5.8011e-13,\n -1.0261e-12, -3.0002e-12, 2.3812e-12, -8.9709e-13, -7.6832e-13,\n -3.8379e-13, -3.2963e-12, 3.5550e-12, -3.1300e-12, 1.7878e-12,\n -5.3578e-13, -6.9718e-13, 5.7939e-13, -1.4210e-12, -1.3197e-12,\n -6.0587e-13, -8.0581e-13, -2.8345e-13, -1.8101e-12, 4.9497e-12,\n -1.9195e-13, 1.8014e-12, -2.4394e-12, 6.5966e-13, 1.1671e-12,\n 2.2444e-13, 2.5970e-12, -7.7295e-13, 2.6897e-13, -9.6172e-13,\n -4.3481e-12, -1.1503e-12, -5.2943e-13, 2.5577e-12, -8.9172e-14,\n -1.3041e-12, 7.2387e-13, -1.6343e-12, -2.4714e-12, 9.6224e-13,\n 2.1250e-12, 1.1120e-12, 2.5088e-12, -6.4887e-13, -7.8163e-13,\n -5.6645e-15, -1.8502e-12, -4.4256e-13, -9.1291e-13, -1.3561e-12,\n 3.3752e-13, -8.7001e-13, 7.6346e-12, -5.4126e-13, -9.2363e-13,\n 1.7663e-12, 2.8349e-13, 1.5436e-12, 1.9369e-12, 1.6254e-12,\n -1.4209e-12, -1.2439e-12, -8.5738e-13, -1.8104e-12, 2.1894e-12,\n 7.4340e-12, 1.4924e-12, -8.5124e-13, 9.7221e-13, 1.4940e-12,\n 3.0818e-14, 8.7755e-13, 1.2239e-12, 3.8245e-13, -1.8545e-12,\n -1.2687e-12, 3.8455e-14, 9.6939e-12, 4.1254e-12, 7.9968e-13,\n -8.6478e-13, 7.9721e-13, -3.6631e-13, -4.2462e-13, -6.7852e-13,\n 2.8218e-12, 7.8638e-13, 4.0000e-13, -3.2977e-12, -1.3295e-12,\n 6.3336e-13, -8.0974e-13, -2.7763e-12, 1.9583e-12, 2.1559e-12,\n -4.1298e-12, -7.5638e-13, -4.9401e-13, -5.4345e-13, 7.4098e-13,\n -1.3459e-12, -9.0916e-13, 6.8672e-13, 4.6853e-13, -4.9273e-13,\n 2.4817e-12, -4.0965e-13, -8.2622e-13, -6.9259e-13, 1.7241e-12,\n 1.4474e-12, -9.9366e-13, -6.2317e-13, -1.7124e-12, 1.8058e-12,\n -2.1600e-13, -2.0253e-12, 3.7776e-12, -7.1092e-13, 6.1425e-12,\n 7.6428e-14, -1.2689e-13, -1.2964e-12, 2.1250e-12, -4.3120e-13,\n -6.2104e-13, -1.6249e-12, -8.3030e-13, -1.1616e-12, 2.2238e-12,\n -5.2706e-13, 1.0290e-12, -9.8619e-13, 1.0861e-12, 1.4834e-12,\n 2.3836e-12, 2.8059e-12, -4.5235e-12, -8.4040e-13, 1.4388e-13,\n -1.0670e-12, -9.1465e-13, -1.8455e-13, 7.5309e-12, -2.2397e-12,\n -7.6189e-13, -6.5331e-13, 2.7136e-12, 1.6576e-12, -1.1029e-12,\n -3.1696e-13, 3.7672e-12, 7.6929e-13, 2.0357e-12, 6.2291e-14,\n -5.1308e-13, -2.0185e-12, -5.5162e-13, -1.1078e-12, 2.2454e-12,\n 9.3831e-12, 8.4865e-13, -7.0726e-13, 5.2781e-13, 3.9125e-13,\n 5.2696e-12, 3.6115e-13, -1.6881e-12, 1.9225e-12, 7.6274e-12,\n -7.9727e-13, 4.6610e-12, -2.1145e-12, -1.4628e-12, 2.9436e-13,\n -8.7243e-13, 1.7368e-12, 1.2248e-12, 8.9016e-13, -2.2171e-12,\n -3.0225e-13, -6.9128e-13, -3.1672e-12, -1.0772e-12, -1.8619e-12,\n -9.1570e-13, 1.8819e-13, 4.0545e-13, -2.4569e-12, -3.4234e-12,\n 5.4810e-13, -7.6704e-13, 1.2775e-13, -8.6345e-13, 8.6204e-13,\n 8.9329e-14, -1.9452e-12, -9.4654e-14, -1.8950e-12, 7.9493e-13,\n 1.5502e-12, 1.9018e-12, -1.3040e-12, 5.1070e-12, -1.1382e-12,\n 4.1723e-12, 4.0288e-13, -2.0288e-12, -6.2395e-13, 1.4531e-13,\n 1.3309e-12, -2.0345e-13, 1.4853e-12, 1.3539e-12, 1.6373e-12,\n 4.8767e-13, 2.7654e-13, -4.8966e-13, -2.5954e-12, -1.5313e-12,\n 9.6419e-13, -1.6579e-14, -1.9830e-12, 1.4164e-13, -8.4116e-13,\n -8.1838e-13, 4.2386e-12, -1.4106e-12, -1.1943e-12, -9.3491e-13,\n -1.2529e-12, -6.2171e-13, -3.5044e-12, 3.0523e-12, 1.2313e-12,\n 2.6050e-12, 1.9340e-12, -1.0797e-12, -8.6229e-13, 2.5187e-12,\n -8.7600e-13, -6.4536e-13, 1.1913e-12, 2.4385e-12, 4.4671e-12,\n -1.8944e-12, -4.9588e-12, 2.9593e-13, 4.6497e-12, -9.2821e-13,\n 1.7437e-12, 6.7410e-13, 1.5286e-12, -4.9707e-13, -1.3394e-12,\n -2.0085e-12, 8.0973e-13, -4.0277e-13, -8.4008e-13, 1.2510e-12,\n -2.3864e-12, -1.2695e-12, 1.7613e-12, -9.1354e-14, -3.3007e-13,\n 2.8510e-12, 1.6713e-12, -2.6246e-12, 2.5351e-13, -3.8400e-13,\n -6.9792e-13, -9.9560e-13, -1.8485e-13, -3.0516e-12, 1.2067e-12,\n -6.3936e-13, 4.7969e-12, -2.9501e-13, -2.8294e-12, 8.6185e-13,\n 5.7745e-12, 1.6248e-13, -1.2818e-12, -9.0886e-12, -5.6043e-13,\n 3.8392e-12, -3.2526e-12, -2.2206e-12, -3.9317e-13, -4.2800e-12,\n -4.2163e-12, 1.8333e-13, -3.1216e-12, -3.1020e-12, -3.5582e-13,\n 2.0885e-12, -2.7218e-12, -4.0556e-12, -8.7852e-13, -8.1631e-13,\n -5.5969e-13, -8.6157e-13, -7.8566e-12, -1.1378e-12, -4.3790e-13,\n -3.2903e-12, -1.3592e-12, -1.3880e-13, 6.0207e-12, -7.8734e-13,\n 3.5334e-13, -1.7360e-12, 7.6272e-13, -5.6387e-12, 5.5147e-13,\n -5.5923e-13, 2.9049e-12, -4.6023e-12, -1.1395e-12, 2.2128e-12,\n 1.0829e-13, 9.8685e-13, -2.1275e-13, -2.2196e-12, -1.6557e-12,\n 4.9183e-12, -1.1088e-12, -5.5415e-13, -3.0816e-12, 2.7286e-12,\n -1.1076e-12, 6.1364e-13, -7.4097e-13, 6.3680e-12, 5.8122e-13,\n 9.7914e-13, 2.4773e-12, 1.1324e-12, -2.2796e-13, -3.6803e-12,\n -1.1465e-12, 1.2175e-12, 1.2033e-12, -7.6906e-13, 2.6321e-14,\n -1.9648e-12, -5.8540e-13, 5.5932e-13, 4.9560e-13, 1.1488e-12,\n -6.1433e-13, 5.9096e-13, -3.8993e-12, 2.5300e-12, 1.8927e-12,\n 3.2613e-13, 3.3243e-12, 3.0590e-13, -1.3537e-12, 1.5471e-12,\n 5.0685e-12, 3.7757e-12, -7.3682e-13, 1.2360e-12, -1.8358e-12,\n 3.0275e-13, -1.0916e-12, -5.2571e-14, 6.2966e-13, -2.3461e-12,\n -1.0170e-12, -1.1127e-12, -3.0282e-13, -9.5400e-13, -1.7122e-12,\n -1.2297e-12, 7.6207e-12, -9.1648e-13, -4.6387e-13, 2.2078e-13,\n 1.8775e-12, -4.0149e-13, 2.8891e-12, -8.9094e-13, -9.0840e-14,\n 2.3796e-12, -8.2007e-13, -6.2837e-14, -9.6506e-13, -1.3125e-13,\n 2.7246e-12, -9.6251e-13, -1.1571e-12, -4.9456e-13, -1.0209e-12,\n -7.2342e-13, -7.7954e-13, -5.0612e-13, -1.9531e-12, 1.2622e-11,\n 1.6816e-13, -1.0210e-12, -1.8120e-12, -8.7202e-13, 2.9801e-12,\n 8.4643e-13, 4.1341e-12, -1.8499e-12, -3.2815e-12, -1.9498e-12,\n -8.7885e-13, -1.1236e-12, 6.5804e-13, 3.9601e-12, -6.5490e-13,\n -9.9033e-13, -4.3697e-13, -8.7589e-13, 2.8096e-12, 3.0290e-12,\n 3.0246e-13, 1.1248e-12, 1.0706e-12, 2.6279e-12, 2.3954e-12,\n -6.7474e-13, -5.6976e-14, 1.4273e-12, 1.4893e-12, -4.0533e-13,\n -2.3002e-13, -2.3732e-12, 3.4419e-13, 1.3517e-12, -3.3972e-12,\n 2.9947e-13, -2.7267e-12, -1.0883e-12, -1.2359e-12, 7.6158e-13,\n -1.2479e-12, -8.8288e-12, -4.0433e-13, 2.3506e-12, -1.0350e-12,\n -1.1404e-12, 1.3039e-12, -8.5412e-13, 4.9741e-13, -2.1598e-12,\n -1.5698e-12, -2.1254e-13, 1.4964e-12, 4.6980e-12, -6.9599e-13,\n 2.1984e-12, 5.1497e-12, -2.3091e-12, 1.9112e-12, -5.6324e-13,\n -2.4502e-12, -3.8288e-12, -4.5397e-13, -4.4108e-12, -2.0227e-12,\n -1.6698e-12, -5.7986e-13, -2.1085e-12, -6.0771e-13, 5.8409e-13,\n -1.2929e-12, -3.2423e-12, -1.6082e-12, -9.3661e-13, 1.4532e-14,\n -8.2198e-13, -5.1137e-12, 5.4380e-12, -1.6538e-12, 5.9186e-12,\n -1.7102e-12, 1.1657e-12, 4.0574e-12, 4.2055e-12, 5.3179e-13,\n 7.8881e-12, 2.5316e-12, -4.1009e-12, 2.4566e-12, 2.8580e-12,\n 1.0768e-12, -4.1248e-12, -2.7651e-12, -3.1533e-12, -1.2264e-12,\n -2.5481e-13, 3.0119e-12, 5.9664e-13, -5.8156e-13, -1.8702e-12,\n -3.5096e-12, -2.5204e-12, -5.5687e-13, -3.5620e-13, -7.6808e-13,\n -1.3462e-12, -8.4754e-13, -2.7665e-13, -9.3108e-12, -1.1812e-12,\n -1.1088e-12, 9.6459e-13, -5.4060e-12, 1.2552e-12, 1.8970e-13,\n 1.5359e-12, -7.5005e-13, -2.1873e-12, 2.2126e-12, 1.0502e-13,\n 2.0127e-12, -2.3861e-12, -1.0915e-12, -7.0952e-13, -1.6290e-12,\n -1.3846e-12, -1.2778e-13, -4.2295e-12, -1.9578e-12, 2.7228e-12,\n 1.6579e-13, -4.4507e-13, 1.7051e-13, -7.7747e-13, 1.8896e-12,\n 1.6806e-12, -1.6751e-12, -6.2542e-13, 7.8877e-13, -8.9798e-13,\n -2.3299e-12, -2.5068e-13, -7.0500e-13, 4.7330e-14, -1.0178e-12,\n 3.9124e-12, -1.0757e-12, -1.1012e-12, -1.5946e-12, -2.1229e-13,\n -1.0637e-12, -1.2847e-12, -3.7203e-13, 2.2856e-12, 3.4091e-12,\n 1.0631e-11, -3.6324e-12, 3.1408e-12, 1.0009e-12, -2.9627e-12,\n 1.9232e-12, -2.1458e-14, -1.6940e-13, 1.2561e-12, 4.1191e-12,\n -7.4956e-13, -6.1044e-13, 6.1287e-12, 1.0851e-12, -2.1695e-12,\n -9.7712e-13, -1.6993e-12, -4.2098e-13, 1.2072e-13, 1.7724e-12,\n -4.7819e-13, -2.1394e-12, -9.1773e-13, 2.5558e-12, -5.6635e-13,\n -7.7921e-13, -1.0087e-12, -3.0924e-13, -3.9260e-12, -5.1830e-13,\n -1.2473e-12, 2.5935e-12, -5.0869e-13, -1.4587e-12, -1.6762e-12,\n -2.5263e-13, -5.4016e-13, 3.2647e-12, 2.1060e-12, 5.0582e-13,\n 1.2256e-12, -3.3450e-13, -5.9762e-13, 5.9384e-12, -2.0131e-12,\n 1.0997e-12, -1.2855e-12, 6.3533e-13, -6.3089e-13, -2.0020e-13,\n -6.1666e-13, -3.8564e-12, -9.6028e-13, -6.1921e-13, -1.5153e-12,\n -9.0455e-13, -1.1742e-12, -1.5883e-12, -2.2803e-12, -1.1314e-12,\n -1.0172e-12, 1.5800e-12, -2.8572e-12, 8.8186e-12, -3.8256e-12,\n -4.5413e-12, 1.4537e-12, 5.2817e-13, -1.1749e-12, -9.3576e-13,\n 3.3613e-12, 2.9620e-12, -5.6755e-12, 5.8753e-12, 2.6163e-13,\n 4.5071e-13, -1.0045e-12, -2.1959e-12, -4.1541e-12, 3.0369e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.4863e-12, 9.2766e-12, 2.2705e-12, 6.7380e-13, 1.7183e-13, 1.2161e-12,\n 6.1693e-13, 5.9595e-13, 1.6602e-13, 8.2799e-13, 1.6522e-12, 2.3596e-12,\n 6.0005e-12, 1.0336e-12, 8.1403e-13, 1.0473e-12, 3.0254e-13, 1.7657e-12,\n 1.7943e-12, 2.7413e-14, 4.9051e-12, 2.7704e-13, 4.7759e-13, 1.1536e-12,\n 1.0047e-11, 3.5176e-12, 4.8992e-12, 7.2878e-13, 2.0815e-13, 9.3842e-14,\n 3.9396e-12, 3.7022e-12, 2.3516e-12, 9.5994e-13, 4.9239e-14, 2.4042e-12,\n 1.2555e-12, 3.0609e-12, 9.1172e-13, 3.7521e-12, 4.4677e-13, 1.3412e-12,\n 2.3805e-13, 1.6577e-13, 3.3022e-12, 1.4582e-13, 6.0006e-12, 7.0141e-12,\n 6.3069e-13, 1.5651e-13, 5.4358e-14, 1.8790e-12, 2.3449e-12, 7.9286e-14,\n 1.1236e-11, 1.4292e-13, 1.4881e-12, 1.6881e-12, 8.3805e-13, 1.1008e-12,\n 1.7087e-13, 7.2114e-12, 3.7695e-12, 2.3416e-13, 3.9406e-12, 1.0024e-13,\n 6.0254e-12, 5.1897e-12, 3.0193e-12, 2.5929e-12, 1.8459e-12, 2.7733e-14,\n 1.4305e-12, 1.3475e-13, 9.1131e-14, 1.4026e-13, 1.1916e-13, 1.1522e-13,\n 6.1827e-12, 2.4081e-12, 2.8594e-12, 8.1688e-13, 2.9167e-12, 8.7713e-13,\n 3.3115e-12, 6.4217e-14, 1.1149e-12, 4.8952e-12, 1.2781e-12, 2.1773e-12,\n 6.2469e-12, 7.1577e-14, 1.8581e-12, 3.2939e-12, 4.9198e-13, 9.2407e-13,\n 3.3662e-13, 2.8817e-12, 1.6961e-13, 2.8129e-12, 9.6824e-13, 3.0247e-13,\n 7.4323e-12, 8.6800e-14, 4.3113e-12, 6.8596e-13, 1.6862e-13, 4.2257e-12,\n 1.7972e-12, 2.7502e-12, 1.3464e-13, 1.4263e-12, 4.1284e-13, 1.8694e-12,\n 2.3368e-13, 1.5134e-13, 1.5195e-13, 4.6002e-14, 3.8597e-13, 5.6537e-12,\n 1.6267e-12, 3.1051e-12, 1.6395e-12, 1.4671e-13, 1.0110e-12, 3.6566e-12,\n 1.1594e-12, 1.6073e-13, 7.9993e-13, 1.2875e-12, 1.5426e-13, 9.1390e-13,\n 1.4079e-12, 7.4612e-14, 9.9397e-13, 2.4596e-12, 8.4848e-12, 3.4789e-13,\n 4.9116e-12, 1.3450e-12, 2.3440e-12, 6.2338e-12, 1.0926e-11, 2.4760e-12,\n 1.8880e-12, 8.7381e-14, 1.6964e-12, 4.8421e-12, 4.6798e-12, 7.7597e-13,\n 9.9008e-13, 3.0352e-12, 2.5036e-13, 1.3179e-11, 1.7121e-13, 2.4133e-12,\n 8.4534e-12, 2.8696e-12, 4.6348e-13, 1.9455e-12, 1.8030e-12, 9.7302e-13,\n 2.5222e-12, 2.4023e-12, 2.0852e-13, 3.1935e-13, 9.6233e-14, 1.0578e-13,\n 3.5973e-12, 1.5285e-13, 1.4234e-13, 2.0104e-12, 7.0851e-14, 3.0596e-12,\n 2.5333e-11, 1.6504e-12, 5.5109e-12, 6.7983e-13, 9.8610e-12, 3.7682e-12,\n 8.1681e-13, 5.8799e-13, 1.1773e-12, 4.1212e-12, 1.9299e-12, 7.4811e-12,\n 3.8137e-12, 3.3161e-12, 1.8266e-12, 9.6755e-14, 5.3539e-12, 8.8593e-13,\n 9.5894e-13, 1.0696e-13, 3.1027e-12, 8.0011e-14, 6.3104e-13, 9.3899e-12,\n 1.5861e-12, 8.7314e-12, 2.5223e-13, 3.5626e-12, 1.7606e-12, 6.8577e-13,\n 2.9308e-13, 3.0504e-14, 2.2953e-12, 2.1998e-14, 8.3078e-13, 6.7349e-13,\n 2.9184e-12, 3.3755e-13, 1.3959e-12, 3.0226e-13, 6.7380e-13, 6.0775e-13,\n 8.2434e-13, 1.0955e-12, 3.6237e-13, 2.2717e-13, 2.3370e-13, 4.4242e-13,\n 3.4205e-12, 6.8542e-12, 1.4811e-12, 5.3537e-12, 1.4746e-12, 2.6576e-13,\n 1.2970e-13, 7.8647e-13, 2.5490e-12, 7.1929e-12, 7.5148e-13, 2.1253e-13,\n 1.9334e-12, 1.1487e-13, 6.8477e-12, 2.8878e-12, 1.3702e-11, 5.3125e-12,\n 7.4324e-13, 4.4006e-13, 4.7762e-12, 3.5334e-12, 1.4927e-12, 2.0575e-14,\n 1.1373e-11, 1.9438e-13, 5.2310e-12, 6.6256e-13, 9.1521e-13, 2.9975e-12,\n 1.5811e-12, 2.2553e-12, 1.8247e-12, 2.2531e-13, 3.7824e-12, 4.8370e-12,\n 4.2061e-14, 8.8865e-14, 1.9933e-12, 1.5762e-12, 1.8576e-13, 5.3453e-12,\n 2.2215e-12, 3.0800e-13, 1.1628e-12, 3.5287e-13, 5.3688e-13, 5.2782e-13,\n 7.4138e-13, 2.4964e-13, 4.3651e-12, 5.8868e-12, 3.1444e-12, 1.8461e-12,\n 1.2246e-12, 3.7960e-12, 2.6457e-13, 1.4852e-13, 2.8248e-13, 6.5361e-13,\n 1.2342e-12, 4.4247e-14, 1.9824e-11, 2.2070e-12, 1.6172e-13, 3.9557e-12,\n 4.0901e-12, 1.1542e-11, 4.2091e-12, 9.0853e-12, 1.3785e-13, 1.1928e-13,\n 2.2240e-12, 2.2138e-12, 6.7258e-12, 1.9367e-12, 6.5494e-13, 5.6630e-13,\n 2.4825e-13, 5.1288e-12, 5.6813e-13, 4.6144e-12, 1.1790e-12, 5.5655e-12,\n 1.1001e-11, 5.3350e-12, 2.2077e-12, 1.4887e-12, 2.2958e-12, 8.2123e-14,\n 2.1440e-13, 4.9304e-12, 2.6718e-12, 5.9461e-12, 8.1552e-14, 1.8456e-12,\n 1.1199e-12, 4.6488e-12, 3.5921e-12, 2.3588e-12, 2.9513e-12, 4.1868e-12,\n 1.8533e-13, 4.6429e-14, 9.2777e-13, 1.1733e-12, 7.6994e-12, 1.4912e-13,\n 9.2853e-12, 3.6573e-13, 1.1327e-12, 4.4004e-13, 7.3342e-12, 6.6471e-14,\n 9.7502e-12, 2.4229e-13, 1.8032e-13, 2.7474e-12, 1.8355e-12, 2.9983e-13,\n 3.4985e-13, 6.9698e-13, 2.3773e-12, 1.1144e-12, 8.1602e-13, 1.1306e-11,\n 1.5782e-13, 7.9916e-12, 3.3347e-12, 1.3949e-12, 4.6470e-12, 2.9367e-12,\n 8.1102e-12, 1.3033e-12, 9.0694e-13, 2.4948e-12, 1.3866e-13, 1.6203e-12,\n 1.3888e-13, 3.3939e-13, 5.7633e-12, 1.5294e-13, 8.3808e-13, 2.3983e-12,\n 3.0345e-12, 2.0821e-13, 2.1403e-13, 3.7197e-12, 2.3044e-13, 1.5239e-13,\n 2.7354e-13, 2.5548e-12, 3.6080e-12, 2.7477e-13, 5.5071e-13, 3.6237e-12,\n 1.0529e-11, 8.3305e-12, 1.6251e-13, 3.7135e-12, 6.5334e-13, 8.8472e-14,\n 4.9836e-12, 1.8966e-13, 2.6621e-14, 1.6160e-12, 2.6857e-13, 4.6849e-12,\n 3.7039e-13, 1.8845e-12, 1.6276e-13, 1.4928e-12, 1.6045e-12, 4.9172e-12,\n 3.2205e-13, 1.0249e-11, 2.0041e-12, 1.3945e-13, 9.7619e-13, 6.0316e-13,\n 3.9820e-14, 2.5257e-12, 5.0081e-12, 8.1952e-13, 9.2646e-13, 4.4249e-12,\n 7.1320e-13, 5.0306e-12, 1.4589e-12, 2.4777e-12, 3.9452e-13, 5.0389e-12,\n 2.0204e-12, 3.1837e-13, 8.4979e-14, 2.6397e-12, 5.7224e-12, 1.0172e-12,\n 2.9031e-12, 2.2677e-12, 1.1622e-12, 3.1913e-12, 1.6639e-12, 4.3719e-12,\n 4.3027e-12, 1.1174e-12, 7.0317e-13, 1.3840e-13, 1.4607e-13, 1.5372e-12,\n 1.7935e-13, 2.3580e-11, 1.7780e-13, 7.7760e-13, 3.7636e-12, 4.4489e-12,\n 1.0227e-12, 4.1560e-12, 1.0157e-12, 7.6790e-12, 5.1776e-12, 3.8014e-13,\n 1.6837e-12, 1.5640e-12, 1.3643e-11, 1.5806e-13, 4.2102e-12, 7.7299e-13,\n 6.6742e-14, 1.1202e-12, 2.7931e-12, 1.2219e-13, 7.3669e-12, 9.5279e-13,\n 6.2278e-13, 5.3630e-12, 1.7638e-13, 2.6996e-12, 1.2269e-13, 4.4629e-12,\n 2.2294e-12, 7.8756e-13, 9.0279e-13, 2.8563e-12, 1.9094e-12, 3.2068e-12,\n 1.4198e-12, 5.1832e-14, 2.6684e-13, 3.4624e-13, 1.0448e-11, 5.9686e-13,\n 5.9522e-12, 4.2188e-14, 5.3305e-13, 1.1425e-12, 2.3261e-13, 2.9354e-13,\n 2.9076e-13, 1.0567e-11, 4.7538e-12, 1.2039e-12, 1.2628e-11, 5.9336e-12,\n 1.7646e-12, 1.3741e-12, 1.7819e-12, 2.7897e-12, 1.6891e-11, 4.6250e-12,\n 2.3868e-13, 4.4418e-12, 1.1152e-12, 2.9981e-12, 7.0320e-12, 5.3712e-12,\n 4.1565e-12, 1.5417e-13, 9.5631e-14, 3.9108e-12, 3.1310e-12, 8.5036e-13,\n 8.1620e-14, 3.4773e-13, 7.3813e-12, 2.7229e-14, 6.1046e-14, 5.6683e-13,\n 2.2898e-13, 1.2607e-13, 2.6218e-12, 1.6558e-11, 7.3057e-12, 7.9564e-12,\n 1.1682e-12, 5.6629e-12, 3.6339e-12, 3.1630e-12, 9.2747e-13, 1.4907e-13,\n 3.9466e-12, 2.0951e-13, 8.8591e-14, 2.3509e-13, 9.0954e-13, 4.8176e-12,\n 3.1618e-14, 4.0738e-13, 1.7996e-13, 6.8551e-13, 2.3057e-12, 6.2211e-12,\n 4.0086e-13, 1.7225e-12, 1.9729e-12, 1.3266e-13, 2.8454e-14, 3.1228e-12,\n 4.9240e-12, 2.8102e-12, 6.0133e-13, 6.5555e-13, 8.9195e-13, 1.7058e-12,\n 1.2642e-12, 2.0148e-12, 1.1767e-13, 5.5908e-12, 4.4541e-12, 1.0644e-12,\n 3.3765e-12, 1.1832e-12, 7.7737e-14, 6.4186e-13, 5.7049e-12, 3.6325e-13,\n 3.2556e-13, 4.9110e-12, 1.5740e-11, 3.7062e-12, 1.5733e-12, 1.4253e-12,\n 1.3007e-13, 6.5168e-12, 2.0272e-12, 5.6405e-13, 2.3840e-12, 4.9454e-12,\n 1.5817e-12, 1.0577e-12, 1.1890e-11, 7.9618e-13, 2.8775e-12, 9.4524e-14,\n 4.0921e-13, 3.6150e-13, 5.6731e-13, 1.1649e-12, 2.3224e-13, 3.5042e-12,\n 8.8496e-14, 2.0226e-12, 7.8610e-14, 4.2924e-14, 6.5203e-13, 3.5225e-12,\n 9.8664e-12, 4.1111e-13, 7.2850e-12, 2.1417e-12, 7.9949e-14, 9.1721e-13,\n 2.3194e-13, 2.2847e-13, 2.9621e-12, 7.4835e-14, 4.3449e-13, 1.4985e-12,\n 1.1198e-13, 6.6845e-13, 9.9082e-13, 1.3988e-12, 3.3668e-11, 5.7994e-13,\n 1.1177e-13, 7.2858e-13, 3.2728e-13, 4.9542e-13, 5.6500e-12, 5.3900e-12,\n 3.4889e-12, 4.3405e-14, 1.5041e-12, 8.1516e-14, 3.5665e-12, 2.5254e-12,\n 4.8655e-12, 3.4202e-14, 1.7426e-12, 2.3360e-12, 9.9521e-13, 1.2350e-11,\n 1.7568e-12, 8.9701e-13, 2.6211e-12, 2.0403e-13, 1.2550e-12, 5.7091e-12,\n 4.9546e-12, 1.5035e-12, 5.6134e-13, 1.0405e-12, 3.8027e-12, 1.8719e-12,\n 2.7994e-13, 4.7927e-12, 3.0558e-13, 6.1327e-14], device='cuda:0')" + }, + "32": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.8195e-14, 7.7869e-15, -7.6575e-15, -5.1930e-15, 6.8542e-15,\n -3.9279e-15, -1.3811e-15, 2.0465e-13, 3.2857e-15, -2.4468e-15,\n 8.0775e-15, -7.5491e-15, 4.2914e-13, 1.6214e-14, -8.4959e-15,\n -7.7708e-15, 1.5700e-15, 2.6903e-14, 1.1239e-14, 1.6027e-15,\n 4.5380e-16, -1.6083e-16, 3.9834e-15, 9.8180e-15, 2.3137e-13,\n -6.6282e-15, 5.4903e-14, 3.7331e-15, 2.3937e-14, 7.5331e-14,\n 3.6874e-14, 9.9752e-14, -4.2372e-15, -1.0408e-14, 8.9701e-15,\n -3.1630e-14, -4.1543e-17, -7.6406e-15, 1.1956e-14, 9.4579e-14,\n 9.7745e-15, 1.4010e-13, -1.6455e-15, -6.3580e-16, 2.8711e-13,\n 6.1888e-15, 2.7564e-14, 1.2821e-13, -5.1286e-15, -1.2656e-15,\n -1.3299e-14, 4.5797e-15, -9.7226e-15, -1.3494e-16, 5.8416e-15,\n 1.0808e-14, -7.2259e-16, 3.6626e-13, 5.1972e-15, 3.5119e-14,\n 4.2359e-16, 4.7698e-14, 3.2627e-14, 4.9557e-15, 3.4268e-14,\n 1.5109e-14, 2.9022e-14, 3.5107e-15, 5.0380e-14, 2.0222e-14,\n 2.1980e-13, 1.6548e-15, 1.2324e-15, 1.0145e-14, 3.6826e-14,\n 2.9432e-15, 1.3482e-14, 6.3927e-15, 5.5347e-14, 2.5363e-15,\n 2.0371e-14, -5.7420e-15, 3.8223e-13, 1.5520e-13, 4.9510e-14,\n -1.1417e-15, 6.4471e-14, -1.0743e-14, 6.3973e-14, 1.6102e-15,\n 1.1396e-13, 1.8327e-14, -7.4096e-15, -4.0876e-15, 2.5195e-17,\n 2.9190e-14, 8.9017e-16, -2.5425e-15, 3.0208e-17, 1.7950e-13,\n -2.4663e-14, 8.8745e-15, -8.3700e-15, -7.5243e-15, -1.5699e-14,\n 2.7261e-15, 6.7481e-16, 3.9997e-14, -1.5563e-14, 8.4757e-15,\n 1.0456e-15, -6.5329e-15, -2.2300e-15, 5.1580e-14, 1.1449e-15,\n -6.1674e-15, -1.9622e-15, 3.0419e-15, 6.5153e-15, 2.9271e-14,\n -1.0623e-14, 4.9151e-15, 2.9677e-13, -6.0739e-15, 2.0837e-13,\n -8.9603e-15, -1.1915e-14, 3.0301e-15, 1.8418e-13, 1.9140e-14,\n -6.9136e-15, 5.5794e-15, 3.4640e-14, 3.8220e-15, 1.1033e-13,\n -1.8899e-15, -5.9896e-15, -6.3507e-16, 2.0704e-13, 2.5378e-14,\n 2.9564e-14, 3.3234e-13, -3.9948e-14, 6.8194e-15, 1.9317e-14,\n 2.7232e-14, 1.7653e-15, 7.2871e-14, 5.2109e-13, 2.4815e-14,\n 1.9299e-14, 2.0450e-14, 6.3609e-15, 1.9300e-13, 3.4490e-14,\n -5.6593e-15, 2.4851e-13, 6.9893e-16, 1.3069e-15, 8.2453e-15,\n 2.5106e-14, 8.6763e-15, 2.9890e-14, 9.8233e-15, 1.6609e-15,\n 2.0262e-13, 2.1461e-13, 8.3313e-15, 3.8130e-14, 2.8238e-14,\n 6.2773e-14, 3.6684e-14, 4.4062e-15, 6.2562e-14, 1.3460e-13,\n 1.1868e-15, 6.5888e-14, 8.6114e-15, 3.5675e-15, 8.3819e-14,\n 1.6170e-14, 9.5814e-16, 1.7940e-14, 3.3070e-14, 4.3011e-14,\n 1.2116e-15, -2.9706e-15, -2.1361e-15, -3.3682e-15, 1.0121e-15,\n -3.1476e-15, 3.9763e-14, 1.7115e-14, -1.5947e-15, -1.2659e-14,\n -4.6397e-16, -6.9065e-15, 5.6222e-15, 5.0635e-16, 8.9980e-14,\n 8.3512e-17, -2.3966e-16, 5.1566e-15, 4.9292e-16, 2.0883e-13,\n -3.4815e-14, -3.8267e-15, 9.2260e-17, 6.0614e-14, -2.9891e-15,\n 2.5205e-13, 5.4221e-15, 1.5198e-15, 4.7246e-15, 2.9337e-15,\n -2.0139e-15, -1.1469e-15, -1.7498e-14, 7.2339e-14, 1.0135e-13,\n 9.9792e-15, -4.2957e-15, -8.1748e-15, 7.8091e-15, -4.0069e-15,\n 6.3015e-14, -6.9728e-15, 2.2417e-14, -1.3528e-14, 4.1730e-16,\n -1.4209e-15, 9.2024e-14, 3.4985e-18, 8.8218e-15, 6.8448e-15,\n -7.0353e-16, 4.5845e-14, -9.6072e-15, 1.5280e-13, 8.1688e-14,\n -1.1131e-15, 2.0882e-15, -6.9551e-16, 8.3300e-14, 6.1306e-14,\n -2.4399e-15, -9.1568e-15, 5.3964e-15, 5.5387e-14, 2.0021e-13,\n 2.5305e-14, -3.8918e-14, -1.1324e-14, 6.2468e-15, 1.0758e-14,\n 5.5246e-15, -1.7303e-14, 5.7937e-14, 1.2642e-14, 3.8209e-14,\n 1.5054e-15, 4.9353e-14, -1.0931e-14, 1.8673e-15, 7.8062e-14,\n -2.7197e-16, 8.2303e-16, -4.5849e-16, -1.0958e-14, 2.8956e-14,\n 1.6368e-13, 7.0093e-14, 5.3971e-14, 2.2083e-13, 1.1253e-13,\n -8.7400e-15, -3.2430e-15, 1.4550e-13, -2.3334e-15, 7.0557e-15,\n -3.6226e-16, 1.0376e-13, -1.0291e-14, 7.4236e-16, -9.8120e-15,\n 3.9499e-13, -1.1889e-15, 2.1073e-16, -1.7449e-13, -4.1964e-15,\n 8.7241e-14, -2.4835e-15, 1.6193e-14, 5.5479e-15, -2.3777e-14,\n -5.1291e-15, 6.5146e-14, -7.3606e-15, -1.3416e-14, -7.8988e-15,\n -1.1750e-15, -8.3257e-16, -1.7931e-14, 1.2345e-14, -3.8514e-15,\n 4.6207e-14, 3.5245e-14, -7.8671e-14, 1.4518e-14, -8.8517e-15,\n -9.3197e-15, 2.5550e-14, 3.5390e-14, 1.0407e-13, -1.7884e-15,\n -1.0474e-14, 1.5167e-15, -2.0195e-14, -6.5240e-14, -1.2872e-14,\n -9.5516e-15, 1.9777e-13, -2.4441e-14, 9.5576e-15, 1.9756e-13,\n -1.1401e-14, -1.3369e-14, -5.8742e-15, 1.0911e-13, 3.2399e-14,\n 3.9536e-14, 2.4904e-14, 2.8314e-14, 8.2183e-15, -3.4679e-14,\n 4.6118e-16, -1.6702e-14, -6.0445e-15, 1.9577e-13, -8.1758e-16,\n 4.4236e-13, 4.3657e-15, 7.4217e-15, 3.6982e-14, -1.8944e-14,\n 1.1025e-15, 2.6055e-14, 6.5184e-14, -4.4980e-16, -1.3280e-14,\n 5.3751e-14, -8.2007e-15, 1.9439e-14, -2.6545e-15, 1.5307e-13,\n -4.5983e-15, 2.9834e-13, -2.1261e-14, 3.4311e-15, 1.3503e-13,\n 1.6917e-14, 2.3625e-14, 3.4424e-14, 3.0210e-15, -3.5990e-15,\n 2.5673e-13, 1.5438e-13, 4.8433e-15, 2.9607e-15, 4.6364e-14,\n 1.6793e-14, 6.9192e-16, -1.0332e-14, 1.0374e-13, -2.3886e-15,\n 5.2228e-14, 3.9067e-14, -9.5277e-15, 1.0595e-13, 2.6904e-14,\n 2.1883e-14, 4.1985e-13, 3.4561e-14, 2.3855e-15, -1.2186e-14,\n -6.4279e-16, 1.4841e-14, 5.1263e-14, 1.8062e-14, 3.0542e-15,\n 7.7517e-14, 2.6153e-14, -9.6111e-15, 6.6061e-15, -4.4671e-15,\n -6.5596e-15, 5.8429e-15, 1.2443e-14, 6.6505e-15, -1.7206e-15,\n 1.5033e-14, -7.9980e-16, 4.3982e-15, 1.2248e-14, 4.3498e-13,\n 3.1071e-14, 1.2764e-13, 3.2238e-14, 1.5695e-14, 3.4020e-13,\n 2.1416e-14, 2.5263e-14, -4.4800e-16, 9.5623e-15, 7.5280e-15,\n -5.0895e-16, 4.7282e-14, -8.2425e-15, 1.3209e-13, -2.4058e-15,\n -4.1143e-15, -9.4221e-15, 5.2756e-16, 3.8420e-14, 1.9065e-14,\n 1.8782e-13, 2.7212e-14, -5.2592e-15, 1.0150e-13, -3.0403e-15,\n 1.6582e-14, -6.8475e-15, 1.1390e-13, 4.6969e-15, 7.3735e-14,\n -1.1199e-14, 3.0682e-14, -1.6993e-14, 2.0892e-14, -1.2545e-14,\n 1.8551e-14, -3.7636e-15, 1.0902e-13, -2.2461e-15, 1.4432e-14,\n 1.1663e-14, -1.7119e-13, -3.7009e-15, 8.1181e-14, 1.8273e-15,\n -3.2765e-15, 4.9957e-14, -1.8326e-15, 2.4939e-14, 3.2675e-14,\n 5.7393e-15, 1.0000e-13, 3.0681e-14, 1.2602e-13, -6.8039e-15,\n 5.1744e-14, 1.6156e-13, 6.5509e-15, 5.4668e-14, 1.9130e-14,\n 7.6499e-15, -2.0534e-14, -2.8682e-15, -3.7679e-14, -8.9142e-16,\n 1.9603e-17, -7.6707e-15, 7.8352e-14, 8.4269e-15, 8.8587e-14,\n -1.9237e-15, -4.7670e-15, 2.8500e-15, 1.6476e-14, 4.4475e-15,\n -1.0594e-15, -5.4146e-14, 6.8568e-13, 7.7834e-15, 5.3785e-13,\n 1.8195e-14, -1.9195e-14, 3.5241e-14, 3.1797e-14, 7.7771e-14,\n 1.0667e-13, 4.4015e-14, -9.2424e-15, 1.1638e-13, 1.0522e-13,\n 2.9352e-15, -1.7587e-15, 4.9723e-14, -5.1992e-15, 4.8533e-15,\n -1.6431e-15, 1.8878e-13, 2.7310e-14, -8.6114e-15, 1.0284e-14,\n 6.9781e-15, 1.3272e-14, 1.4681e-14, -5.8096e-15, -2.8717e-15,\n -9.3431e-17, -5.8232e-16, 1.2528e-14, -2.2165e-13, 1.3185e-14,\n 1.6716e-15, 3.6849e-14, -6.0827e-14, 7.4663e-14, 1.1896e-13,\n 1.4761e-14, 1.3555e-14, -2.8155e-17, 1.6986e-15, 1.0509e-14,\n 3.0214e-15, -3.0592e-15, -7.8842e-15, 3.3731e-15, 2.6898e-17,\n -3.1791e-15, -1.4114e-14, -2.3582e-14, 6.4018e-14, 9.1387e-14,\n 1.9886e-14, 8.5018e-15, 1.4449e-14, 7.9180e-15, 5.6386e-14,\n 1.8592e-14, 1.1047e-13, 1.6627e-14, 2.2813e-14, 1.4765e-14,\n 2.2993e-14, -1.1689e-14, 8.5254e-15, 8.2877e-15, 2.1153e-14,\n 2.8684e-13, 4.2634e-14, 9.0090e-15, 2.9131e-14, 1.0417e-15,\n -2.9568e-15, 9.3241e-16, -9.4661e-15, 3.4914e-15, 7.0231e-14,\n 3.6494e-13, -7.5862e-15, 6.3176e-14, 1.1854e-13, -3.2026e-15,\n 1.7506e-13, -1.0315e-14, 8.2986e-15, 2.9319e-14, 3.0251e-13,\n -1.5961e-15, 2.8787e-14, 9.0388e-14, 8.1001e-14, -3.4834e-15,\n -4.0369e-16, 1.5538e-16, -5.1275e-15, -4.1045e-15, 9.0043e-16,\n 3.8990e-15, -1.4771e-15, 1.7210e-14, 8.5108e-14, -1.3871e-15,\n -2.9744e-15, -1.6156e-15, -1.2235e-14, -2.4029e-14, 2.5984e-14,\n -3.6849e-15, 1.2457e-13, 2.3997e-14, 6.6830e-15, -1.5887e-15,\n 6.3287e-14, 3.5506e-14, 4.0711e-14, 6.0529e-15, -1.0379e-14,\n 8.0881e-15, -3.0622e-15, -3.9009e-15, 1.9051e-13, 1.6305e-15,\n 9.9790e-14, 1.6085e-15, -8.6231e-15, 5.0787e-15, 2.8915e-14,\n 7.5119e-14, -1.6746e-14, 5.0546e-16, 7.2153e-15, 1.1042e-14,\n -2.2156e-15, 1.4026e-13, 1.2872e-13, 2.2992e-14, 1.6477e-14,\n 1.0218e-15, 1.3828e-13, -4.4510e-15, 2.2121e-13, -1.2014e-14,\n -3.3761e-14, -9.2295e-15, 5.0293e-14, 6.5490e-15, 2.5069e-15,\n 2.1141e-13, 9.4864e-14, -4.8600e-14, 1.9432e-13, 6.4904e-14,\n -7.5023e-15, 1.8395e-15, 9.7177e-16, -2.5924e-14, 3.2745e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.8317e-15, 1.1762e-14, 2.8740e-15, 7.4498e-17, 2.5618e-17, 6.3583e-16,\n 3.0922e-16, 3.0782e-16, 2.2326e-16, 6.1594e-17, 1.6815e-17, 1.6886e-15,\n 1.3067e-14, 4.5306e-16, 3.0776e-16, 3.8111e-17, 4.0156e-17, 1.2407e-16,\n 9.5905e-16, 4.2313e-18, 3.5870e-15, 9.2423e-18, 1.3839e-17, 7.1494e-17,\n 8.1967e-15, 3.3973e-15, 7.9993e-16, 2.7579e-16, 1.1195e-16, 5.5299e-17,\n 1.7486e-15, 1.1437e-15, 5.3656e-16, 1.6177e-16, 1.9567e-17, 2.1378e-15,\n 3.3929e-16, 4.8468e-16, 6.8331e-17, 4.1147e-15, 2.9480e-16, 1.4144e-15,\n 1.2533e-17, 1.4639e-17, 8.2021e-15, 3.5973e-17, 2.0269e-16, 2.0460e-15,\n 1.5437e-16, 9.5319e-17, 2.1113e-17, 4.3069e-16, 9.7960e-16, 1.9074e-17,\n 2.3514e-15, 1.1335e-17, 8.4835e-17, 1.9428e-15, 1.8528e-17, 2.2857e-17,\n 6.0668e-17, 5.4994e-15, 1.1226e-15, 1.2638e-16, 2.1226e-15, 6.9240e-18,\n 6.8948e-15, 1.5202e-15, 1.5430e-15, 1.8365e-15, 2.4441e-15, 3.4994e-18,\n 6.3523e-17, 1.0020e-16, 3.8702e-17, 1.1380e-16, 1.9513e-18, 7.3986e-17,\n 2.0935e-15, 6.3186e-17, 3.2351e-15, 4.0467e-17, 4.5784e-15, 1.0746e-16,\n 9.5840e-16, 4.1627e-17, 1.9636e-16, 7.7606e-16, 3.1157e-16, 1.0544e-14,\n 7.0329e-15, 2.0208e-17, 8.0413e-16, 2.5337e-16, 7.7282e-17, 4.3157e-16,\n 1.6439e-17, 3.5479e-16, 6.6216e-17, 5.1146e-16, 1.0212e-16, 1.4653e-17,\n 1.2214e-14, 3.5630e-17, 4.3983e-15, 2.5257e-16, 1.9586e-17, 3.9671e-15,\n 1.4588e-16, 4.4295e-15, 9.5943e-17, 4.7704e-16, 6.3814e-17, 2.0630e-15,\n 5.3194e-17, 9.2940e-17, 5.0989e-18, 1.7447e-17, 1.5873e-17, 1.1683e-15,\n 2.0064e-15, 4.0059e-15, 3.1090e-16, 1.0076e-16, 2.5132e-16, 2.9448e-15,\n 6.5766e-16, 7.5821e-18, 3.3674e-17, 7.2372e-17, 2.2209e-17, 6.2649e-18,\n 1.4124e-17, 5.2248e-18, 9.3591e-17, 6.5790e-16, 4.2173e-15, 4.0447e-18,\n 4.6610e-15, 1.7504e-16, 9.0272e-16, 5.9368e-15, 2.3359e-14, 1.5847e-15,\n 1.7923e-16, 4.1444e-17, 4.2844e-16, 2.6150e-16, 4.6288e-15, 1.1327e-15,\n 3.2217e-16, 2.0837e-15, 1.2076e-17, 1.0415e-14, 5.2111e-17, 4.5929e-16,\n 5.7059e-15, 3.8147e-16, 1.1194e-17, 6.0781e-16, 5.0233e-17, 2.0060e-16,\n 1.9911e-15, 3.4323e-16, 6.9567e-17, 5.3081e-17, 1.2904e-17, 5.0801e-17,\n 9.6634e-17, 8.8592e-17, 1.0175e-16, 1.3271e-15, 8.9935e-17, 1.7282e-15,\n 9.4663e-15, 7.0626e-16, 1.9870e-15, 1.9180e-17, 1.8160e-14, 4.8795e-15,\n 8.7679e-17, 4.7611e-17, 1.7163e-16, 5.8474e-15, 3.4624e-16, 1.1204e-15,\n 2.7509e-15, 3.8808e-15, 4.0359e-15, 8.8626e-18, 6.9622e-15, 1.5263e-17,\n 3.1738e-16, 3.5641e-17, 4.6950e-15, 2.9936e-17, 4.2241e-18, 9.5739e-15,\n 1.5358e-17, 5.6982e-15, 4.1454e-18, 1.0553e-15, 1.3954e-15, 5.7384e-17,\n 3.2850e-16, 8.6110e-17, 3.7717e-15, 5.2483e-17, 4.8448e-17, 9.4343e-16,\n 3.8545e-15, 2.4434e-17, 1.2395e-15, 4.5583e-18, 2.4382e-16, 1.0357e-17,\n 1.6583e-16, 5.0583e-16, 1.1347e-17, 4.8024e-17, 4.7151e-17, 1.3085e-17,\n 8.0721e-16, 9.5578e-15, 1.1049e-16, 2.3645e-15, 4.0219e-16, 3.1119e-18,\n 1.1059e-17, 5.6402e-18, 4.4763e-16, 2.6823e-15, 2.7877e-17, 1.0341e-16,\n 9.8344e-16, 2.6298e-17, 4.7570e-15, 2.9797e-15, 8.8014e-15, 6.8497e-15,\n 2.0047e-16, 8.3993e-18, 2.0864e-16, 7.5566e-16, 5.0237e-16, 1.2256e-18,\n 9.5010e-15, 5.4208e-17, 3.8865e-15, 4.6286e-17, 1.2876e-15, 2.8188e-15,\n 4.8804e-16, 2.4553e-16, 8.5890e-17, 7.2510e-17, 2.3606e-15, 6.5185e-15,\n 4.1579e-17, 8.7659e-18, 7.8275e-15, 3.6337e-16, 1.1316e-17, 7.2321e-16,\n 1.4487e-15, 3.2945e-17, 3.5269e-16, 2.2520e-17, 7.8647e-17, 2.8320e-16,\n 5.5495e-16, 1.3367e-17, 9.9326e-15, 9.9027e-15, 1.6744e-15, 3.2950e-16,\n 3.6574e-16, 1.6437e-15, 5.1408e-17, 1.3168e-16, 9.3810e-18, 8.5629e-18,\n 8.0181e-17, 3.6050e-17, 1.1431e-14, 2.2223e-16, 1.9039e-17, 4.2663e-16,\n 4.2766e-15, 7.8446e-15, 1.5340e-15, 1.2705e-15, 1.9832e-17, 7.4272e-17,\n 2.9098e-15, 1.6612e-15, 3.1702e-15, 3.5005e-16, 1.7837e-17, 1.1722e-16,\n 2.2864e-17, 1.4320e-14, 6.1317e-18, 5.7293e-16, 3.3624e-16, 3.5247e-15,\n 7.3107e-15, 7.3484e-16, 8.4036e-16, 5.7396e-17, 1.1141e-15, 5.7882e-17,\n 1.1639e-17, 8.6213e-16, 8.1090e-17, 8.4717e-16, 8.4975e-17, 5.5513e-16,\n 7.9099e-16, 5.4806e-15, 8.8602e-16, 2.0257e-15, 2.6063e-15, 8.0838e-15,\n 4.3498e-17, 2.0889e-17, 2.0457e-16, 4.0532e-17, 1.2760e-14, 1.2324e-16,\n 6.8292e-15, 1.4490e-16, 3.6512e-17, 8.2451e-17, 3.4001e-15, 3.1418e-17,\n 2.7740e-15, 3.4779e-18, 2.3998e-17, 3.7569e-17, 6.4908e-15, 2.1314e-17,\n 2.1672e-17, 2.9720e-16, 1.3087e-15, 1.9791e-16, 4.9992e-16, 1.1232e-14,\n 2.4548e-18, 3.5307e-15, 1.7996e-15, 3.1672e-16, 2.8429e-15, 1.9641e-16,\n 2.1143e-15, 3.6184e-16, 1.5091e-16, 2.6206e-15, 5.0647e-17, 1.1302e-15,\n 1.1627e-16, 1.8456e-17, 5.4469e-15, 7.7585e-18, 1.0599e-15, 4.9054e-15,\n 2.4520e-15, 1.3706e-16, 7.1533e-17, 1.8582e-15, 5.5277e-17, 5.6867e-17,\n 5.8079e-18, 3.3701e-15, 3.2507e-15, 2.1681e-16, 6.3716e-18, 7.9697e-16,\n 5.8639e-15, 2.4427e-15, 2.8120e-18, 8.3859e-15, 2.7566e-16, 2.8884e-18,\n 4.0874e-15, 1.9976e-16, 6.4246e-17, 8.3999e-16, 5.7675e-18, 3.1375e-15,\n 6.0525e-17, 2.2242e-16, 6.1850e-18, 1.8156e-16, 6.5337e-16, 1.4377e-15,\n 5.0467e-17, 1.0024e-14, 2.4339e-15, 8.2987e-18, 2.4193e-17, 1.6199e-17,\n 2.9588e-17, 1.0650e-15, 9.1705e-15, 4.2816e-16, 4.4242e-16, 6.4511e-15,\n 2.3456e-17, 3.3998e-15, 8.4687e-16, 1.4817e-15, 5.5617e-17, 6.2683e-16,\n 1.7111e-15, 1.0189e-17, 3.9992e-17, 3.3622e-16, 4.3293e-15, 4.8471e-16,\n 2.8525e-15, 2.6441e-15, 3.1047e-16, 6.6734e-15, 3.0972e-16, 2.2519e-15,\n 4.6345e-15, 4.9836e-17, 7.1538e-17, 2.9796e-16, 1.1759e-16, 2.0275e-16,\n 1.3382e-17, 1.3590e-14, 1.9616e-17, 1.1990e-16, 2.9332e-16, 6.0010e-15,\n 1.1407e-17, 8.5903e-16, 6.8752e-17, 8.2009e-15, 4.7381e-15, 2.4113e-18,\n 2.9797e-17, 5.1145e-16, 3.5726e-14, 1.2848e-16, 2.5642e-16, 1.0810e-16,\n 5.3648e-17, 2.1230e-15, 7.4440e-16, 2.9671e-17, 1.0055e-14, 5.9880e-17,\n 2.4739e-16, 9.3127e-15, 1.0905e-16, 4.6823e-16, 8.0070e-17, 1.5101e-15,\n 9.6672e-16, 1.3451e-16, 4.4254e-17, 3.8540e-16, 1.7249e-15, 7.8877e-16,\n 1.2810e-15, 8.4818e-17, 1.7354e-17, 2.5027e-16, 7.3939e-15, 7.0284e-18,\n 1.1755e-14, 2.0334e-17, 5.0111e-17, 9.4732e-17, 1.6015e-17, 1.0918e-17,\n 1.9429e-17, 6.6005e-15, 2.7398e-15, 1.1077e-17, 1.4279e-14, 1.0042e-14,\n 6.5520e-16, 9.4608e-16, 5.8355e-16, 5.4497e-16, 3.7172e-14, 8.9841e-17,\n 1.7642e-17, 7.3166e-15, 4.5977e-16, 3.4501e-15, 1.3582e-14, 1.9813e-15,\n 2.7117e-15, 3.9160e-17, 2.0924e-17, 2.5050e-15, 1.0449e-15, 4.1872e-18,\n 7.1693e-17, 4.6608e-17, 4.7069e-15, 4.9366e-17, 4.1300e-18, 3.1224e-16,\n 2.8733e-17, 4.0030e-18, 3.0131e-16, 7.7774e-15, 2.2170e-15, 1.5467e-14,\n 8.7550e-17, 1.7581e-15, 1.1781e-15, 6.9684e-16, 9.5756e-18, 5.9700e-18,\n 1.5207e-15, 3.0139e-17, 6.2144e-17, 6.0082e-17, 5.5576e-17, 7.6831e-16,\n 5.8712e-17, 1.0239e-16, 1.5596e-17, 9.8665e-17, 4.1107e-16, 2.5111e-15,\n 1.9292e-17, 3.7974e-16, 1.2430e-15, 8.3836e-17, 5.9431e-17, 2.1799e-15,\n 2.5633e-15, 1.4950e-15, 5.0175e-18, 3.9162e-16, 5.7519e-16, 1.3676e-15,\n 7.7209e-17, 5.3357e-16, 8.9897e-18, 6.9852e-15, 2.6142e-15, 5.5533e-16,\n 4.7398e-15, 2.6177e-16, 7.6451e-17, 4.5308e-17, 3.1473e-15, 1.3114e-17,\n 4.4987e-17, 3.6113e-15, 1.9322e-14, 9.6819e-15, 6.9914e-16, 1.1898e-16,\n 1.6268e-17, 6.3270e-16, 3.1255e-16, 8.0331e-17, 1.6813e-16, 3.3037e-15,\n 6.3837e-16, 1.2944e-16, 1.8607e-14, 2.6413e-17, 2.2490e-15, 8.1583e-18,\n 2.5774e-17, 1.6733e-17, 2.0554e-17, 2.5106e-16, 8.9332e-17, 3.2966e-15,\n 2.3097e-17, 5.1512e-16, 4.5546e-17, 2.0790e-17, 1.1240e-16, 1.0601e-15,\n 1.8679e-14, 1.3415e-17, 9.0342e-16, 7.0753e-16, 3.0060e-17, 2.1534e-16,\n 5.4124e-18, 3.6090e-17, 2.6687e-15, 3.2883e-17, 9.3692e-18, 1.2672e-16,\n 3.5972e-17, 6.1918e-17, 5.4739e-16, 3.7335e-16, 1.2515e-14, 1.5365e-17,\n 1.0818e-16, 6.4231e-17, 1.9922e-17, 1.7724e-17, 5.3558e-15, 1.0699e-15,\n 1.0851e-16, 9.4681e-18, 1.2707e-16, 7.3180e-18, 9.8641e-16, 4.3048e-15,\n 3.5899e-15, 4.5933e-18, 1.0748e-15, 1.2863e-15, 2.4483e-16, 1.0812e-14,\n 8.2768e-16, 3.2124e-17, 6.5664e-16, 8.0903e-18, 4.7383e-17, 5.1936e-15,\n 2.7123e-15, 5.7403e-16, 5.6275e-16, 2.9050e-16, 9.7258e-15, 1.2221e-16,\n 9.4155e-17, 6.9491e-15, 3.7709e-17, 3.6748e-17], device='cuda:0')" + }, + "33": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.7717e-14, 6.4530e-14, 1.0579e-13, 1.2363e-14, -5.4416e-15,\n 1.3259e-14, -2.3819e-14, 1.6296e-13, -2.5881e-15, 6.8450e-15,\n -7.2208e-15, -9.0975e-15, 2.2027e-13, 1.0544e-14, 9.5870e-14,\n 2.7043e-14, 6.1570e-15, 7.7858e-14, 2.4415e-14, -4.1655e-15,\n 3.3554e-14, 3.9212e-14, -3.1686e-15, 2.2822e-14, 2.2292e-13,\n 3.6451e-14, 1.1713e-13, -2.2479e-14, -1.0579e-14, 1.0913e-13,\n 7.5152e-14, 1.4029e-13, 2.1183e-14, 5.1379e-14, -9.4105e-15,\n -3.8513e-14, -2.3151e-15, 1.4577e-14, 1.2416e-13, 8.2476e-14,\n 3.0489e-14, 1.1396e-13, 4.3231e-15, -1.2579e-14, 1.4854e-13,\n -2.0552e-15, 9.2725e-14, 1.5086e-13, 2.0254e-14, 4.2006e-15,\n 3.3691e-14, 8.9289e-15, 1.7011e-14, -3.0865e-16, 3.2256e-14,\n -5.9640e-15, 9.5993e-16, 3.0028e-13, -5.2139e-15, -2.3775e-14,\n -2.6188e-16, 7.9731e-14, 1.0163e-13, -1.6903e-15, 1.0739e-13,\n 2.7116e-14, 4.1066e-14, -2.1123e-16, 4.2889e-14, 1.1555e-13,\n 2.8072e-13, 8.8679e-14, 4.5539e-15, -4.3550e-15, 1.0461e-13,\n -1.9638e-15, 7.9321e-14, -2.6665e-15, 8.3349e-14, 1.4155e-14,\n 3.6322e-14, 4.8436e-14, 3.5259e-13, 1.9211e-13, 8.1447e-14,\n 3.2321e-15, 9.6841e-14, 2.1002e-14, 6.9642e-14, 3.3751e-14,\n 1.5572e-13, 8.0026e-14, 4.5467e-15, -3.7991e-14, -4.6633e-16,\n 8.0131e-14, -1.7383e-15, -5.2805e-15, 1.2950e-17, 1.5267e-13,\n -2.6993e-14, 4.0133e-14, 1.4688e-14, 1.2873e-14, 6.2948e-14,\n -5.9474e-15, -1.8556e-15, 8.5286e-14, 5.2177e-14, 4.1922e-14,\n -3.2376e-16, 2.4123e-14, 3.1286e-15, 6.2924e-14, 9.9500e-14,\n 2.5570e-15, 2.7755e-15, -2.7358e-15, 1.6798e-14, 1.0959e-13,\n 5.0008e-14, 1.2831e-14, 2.0804e-13, 1.1548e-14, 2.5032e-13,\n 6.4702e-15, 3.6334e-14, -5.5309e-15, 1.5256e-13, 5.3081e-14,\n 2.1786e-14, 1.8165e-14, -2.3142e-14, 2.4490e-14, 1.4059e-13,\n 1.1622e-14, 7.7656e-14, 7.6189e-16, 1.3261e-13, 1.0086e-13,\n 1.2376e-13, 1.9131e-13, -3.7165e-14, -4.6160e-15, 6.4889e-14,\n -2.1824e-14, -8.5586e-16, 7.8334e-14, 3.1518e-13, 3.3496e-14,\n -1.5196e-14, -1.4737e-14, 1.2930e-13, 1.4478e-13, -2.6430e-14,\n 2.3962e-14, 1.9960e-13, 7.1526e-14, -4.4728e-16, 4.4510e-14,\n -1.6786e-14, 1.7782e-14, 5.4541e-14, -1.1212e-14, -5.6939e-16,\n 3.2861e-13, 1.3317e-13, -7.0676e-15, 8.2657e-14, -1.3656e-14,\n 2.0399e-13, 8.4732e-14, -1.2162e-14, 1.2195e-13, 2.7685e-13,\n -1.6345e-15, 1.8985e-13, 1.5289e-14, 1.7937e-14, 9.2288e-14,\n -1.4133e-14, 9.9684e-14, 9.1152e-14, 7.9303e-14, 3.3929e-14,\n 3.8600e-14, 2.8018e-14, 5.1967e-18, 1.2493e-14, -8.6154e-15,\n 2.6745e-14, 6.8798e-14, 6.8763e-14, -1.5094e-14, -8.3724e-15,\n 4.4779e-16, 1.7083e-14, 5.7559e-14, -6.9900e-16, 1.0597e-13,\n 5.0903e-14, -8.2296e-15, 4.9076e-14, -1.8362e-15, 1.3008e-13,\n 1.7611e-14, 1.0028e-13, 2.1331e-16, 2.0213e-13, 8.7029e-15,\n 2.0870e-13, 4.8851e-14, 4.1178e-15, 3.5519e-14, 5.7398e-14,\n 7.6525e-14, 4.2817e-14, 7.9907e-14, 1.1225e-13, 1.2614e-13,\n -5.0756e-15, 5.3205e-14, 1.8757e-14, 9.7730e-15, -3.1446e-15,\n 9.7896e-14, 4.5066e-14, 2.7155e-14, 3.9777e-14, 6.6251e-16,\n 2.7193e-14, 1.8250e-13, -6.2481e-16, -1.2335e-14, 3.2091e-14,\n 2.8553e-15, 6.1856e-14, -1.2398e-14, 1.6923e-13, 1.1016e-13,\n 1.2221e-13, -7.1958e-16, 2.2050e-15, 7.2734e-14, 1.3641e-13,\n 3.9794e-15, 1.3950e-14, -2.3381e-15, 1.3199e-13, 2.0849e-13,\n 3.2118e-14, -6.2315e-14, 3.9148e-14, 1.7794e-13, -1.0023e-14,\n 9.5551e-14, 5.9776e-14, 1.1014e-13, -9.7354e-15, 4.6280e-14,\n 1.2073e-14, 9.2073e-14, 2.2210e-14, -1.1504e-15, 1.0918e-13,\n -3.9544e-15, 8.2690e-15, 2.3141e-16, 3.8387e-14, -1.7650e-14,\n 1.6462e-13, 1.1842e-13, 3.3653e-14, 1.2371e-13, 8.7546e-14,\n 1.2557e-14, 1.7953e-14, 9.8944e-14, -3.3566e-14, -3.0632e-15,\n 4.2169e-16, 2.0137e-13, 3.2061e-14, -3.7449e-15, 7.0728e-14,\n 2.6218e-13, 7.9299e-16, 4.9269e-17, -1.5711e-13, 3.9384e-14,\n 1.7426e-13, -6.3905e-15, 1.9062e-14, -5.4948e-15, -1.6978e-14,\n -6.2512e-14, 8.5562e-14, -2.5782e-14, -1.8236e-14, 3.4401e-14,\n 3.9492e-16, 5.8444e-15, -4.5627e-14, -1.2164e-14, 8.8042e-15,\n 6.4868e-14, 5.2808e-14, -1.4594e-13, -1.4127e-14, 2.0849e-14,\n -1.7149e-14, -2.4074e-14, 6.6899e-14, 2.2490e-13, 3.8571e-15,\n 5.2553e-14, 7.9854e-15, 6.0639e-14, -7.1388e-14, 5.7236e-14,\n 1.8552e-14, 1.7394e-13, -1.7071e-14, 3.2865e-14, 1.5615e-13,\n 4.1968e-14, 6.8488e-14, 2.2083e-14, 5.7515e-14, -3.2130e-14,\n 1.9373e-13, -2.1358e-14, 5.1006e-14, 7.6435e-15, 1.1362e-13,\n -1.2082e-15, 5.7186e-14, 1.0028e-14, 2.5588e-13, 6.6493e-14,\n 1.7031e-13, -1.4223e-15, -3.2252e-15, 6.1781e-14, -2.0423e-14,\n -9.4595e-16, 9.4805e-14, 1.0293e-13, 2.0703e-15, 4.4078e-14,\n 4.5059e-14, 1.5329e-14, 7.3167e-14, 6.2805e-14, 1.2699e-13,\n 4.6421e-15, 1.4147e-13, -2.3418e-14, -1.0594e-15, 1.3919e-13,\n -8.9454e-15, 1.4752e-13, 7.5384e-14, -9.3956e-15, 8.9494e-14,\n 2.2879e-13, 1.8497e-13, -4.8116e-15, -1.3272e-15, 3.8870e-14,\n -8.9401e-15, 2.6850e-16, 3.4606e-14, 1.0553e-13, -9.2079e-15,\n -3.3048e-14, 4.6870e-14, 2.0570e-14, 7.5374e-14, 3.3413e-14,\n 3.7578e-14, 3.0618e-13, -2.4674e-14, -1.6017e-15, 5.0475e-14,\n 1.8059e-16, -1.0731e-14, 1.4419e-13, -1.5552e-14, 5.0641e-14,\n 1.3697e-13, 4.4139e-14, 7.4253e-15, 3.1808e-14, 4.1681e-14,\n 1.2355e-13, -6.8191e-15, 3.3173e-14, 4.0882e-14, 2.6577e-15,\n -1.2259e-14, 3.3365e-15, 1.6894e-14, 2.4839e-14, 4.3160e-13,\n 7.1168e-14, 8.0444e-14, 3.4875e-14, 3.9301e-14, 1.9477e-13,\n 8.1017e-14, 1.6994e-13, -1.9399e-15, 8.8173e-15, 1.3799e-14,\n 4.8477e-16, -3.1609e-14, 6.2364e-14, 1.8458e-13, 1.8621e-14,\n 1.1519e-14, 2.1411e-14, -2.9657e-16, 1.3769e-13, 1.3893e-13,\n 1.1741e-13, 9.2718e-14, 7.7188e-14, 1.4759e-13, 9.5303e-16,\n -1.3012e-14, 4.5245e-14, 1.2519e-13, 9.2797e-14, 7.6133e-14,\n 3.0734e-14, 2.5639e-14, 4.7792e-14, 9.6182e-14, -1.7819e-14,\n 6.8737e-14, -1.0358e-14, 7.4745e-14, 8.3286e-15, 7.8732e-14,\n 2.6361e-14, -1.4996e-13, 1.3945e-14, 1.3773e-13, -3.2862e-15,\n 1.3297e-14, 1.0400e-13, 4.7762e-16, 7.5605e-14, 3.1194e-14,\n 1.5334e-14, 8.8294e-14, 1.0093e-13, 2.0041e-13, 1.9662e-14,\n 1.2533e-13, 2.1888e-13, 8.1813e-15, 1.1779e-13, -1.2111e-14,\n 1.1982e-14, -2.5809e-14, 1.5937e-14, -3.7116e-14, -5.3874e-15,\n -1.3971e-15, 1.4455e-14, 4.9873e-14, 3.9265e-14, 9.8599e-14,\n 3.8194e-15, -2.9061e-14, 1.8179e-14, 4.0929e-14, 5.4619e-14,\n 2.6237e-15, -5.3034e-14, 2.8580e-13, 1.7569e-14, 2.8126e-13,\n 2.5464e-14, 6.8415e-14, 1.6941e-13, 1.7173e-13, 9.8889e-14,\n 2.8069e-13, 1.3208e-13, -5.7954e-15, 1.4670e-13, 1.5483e-13,\n 8.4170e-14, -7.5942e-16, 2.9434e-14, -3.3623e-14, 2.4533e-14,\n 1.4823e-15, 1.7388e-13, 9.1872e-14, 2.1468e-14, 1.6433e-14,\n 4.6728e-15, 8.7259e-15, -1.1447e-14, 3.3537e-14, 5.6937e-15,\n 1.0915e-14, 1.6494e-15, 4.7753e-14, -1.5280e-13, 3.2179e-14,\n 2.2259e-14, 9.1757e-14, -6.4041e-14, 1.1011e-13, 9.9406e-14,\n 9.9276e-14, -1.1605e-14, 9.8266e-16, -5.6632e-16, 6.0729e-14,\n -1.0270e-15, -1.1361e-14, 1.2029e-14, 1.3895e-15, 8.7885e-15,\n 7.5968e-15, 3.2650e-14, -1.7940e-14, -4.8588e-14, 1.4803e-13,\n 6.6211e-14, 3.9300e-14, -8.3008e-15, -7.7071e-15, 1.1950e-13,\n 1.0412e-13, 6.3602e-14, 4.4694e-14, 8.2912e-14, 4.1931e-14,\n 1.8643e-14, 2.9509e-14, -7.7647e-15, 5.6977e-14, 4.1619e-14,\n 2.0806e-13, -2.9662e-14, 3.2541e-14, 3.5398e-14, -6.9646e-16,\n -2.5024e-16, 2.2105e-15, 2.6248e-14, -1.1632e-15, 1.5967e-13,\n 3.7412e-13, -7.3211e-15, 1.5094e-13, 1.1619e-13, -2.2392e-14,\n 1.4772e-13, 4.1984e-14, 5.0994e-14, 9.7080e-14, 2.1737e-13,\n 6.7927e-15, 5.3838e-14, 2.3076e-13, 1.0869e-13, -5.6746e-15,\n 7.3172e-16, -1.4520e-16, 1.9616e-14, 4.7753e-14, 9.8082e-14,\n -3.1297e-15, -6.0761e-15, -1.4686e-14, 1.4339e-13, 9.5867e-15,\n 4.5285e-15, 3.6829e-15, 2.5119e-14, -3.4910e-14, -1.7265e-14,\n 6.0211e-15, 1.4828e-13, -1.3943e-14, 2.4604e-14, 8.9498e-15,\n 7.0052e-14, 5.9930e-14, 1.4670e-13, -2.0056e-15, 5.7821e-14,\n -3.2449e-15, 2.6454e-14, 2.1324e-14, 2.4195e-13, 4.4051e-15,\n 1.1946e-13, -2.5124e-15, 6.2939e-14, -4.6187e-15, 6.0611e-14,\n 6.9252e-14, -1.6043e-14, 2.6418e-14, -6.0571e-15, 2.5616e-14,\n 1.5991e-14, 7.6257e-14, 7.0257e-14, 2.1663e-14, -1.3151e-14,\n 2.4562e-14, 1.3218e-13, -3.7539e-15, 3.1688e-13, -1.2107e-14,\n -4.7934e-14, 8.5505e-14, 8.5718e-14, -6.0839e-15, 3.5368e-16,\n 1.8613e-13, 1.5470e-13, -8.3865e-14, 2.3966e-13, 8.6122e-14,\n 5.3888e-14, 2.2297e-14, -8.4740e-16, -3.8848e-14, 1.9472e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.4010e-15, 5.6914e-15, 2.2190e-15, 7.3527e-16, 6.7213e-18, 1.0330e-15,\n 5.0186e-16, 5.7733e-16, 7.5093e-17, 6.5747e-16, 1.0892e-15, 2.0481e-15,\n 2.7412e-15, 9.3252e-16, 6.0154e-16, 4.6407e-16, 2.7506e-16, 1.3240e-15,\n 1.4327e-15, 3.0931e-17, 3.9302e-15, 1.9135e-16, 4.1371e-16, 5.4482e-16,\n 4.3458e-15, 2.3804e-15, 3.3758e-15, 9.0475e-16, 2.6562e-17, 2.5813e-17,\n 2.9848e-15, 3.1016e-15, 2.3632e-15, 6.6594e-16, 2.3915e-17, 1.9276e-15,\n 1.2372e-15, 2.3930e-15, 1.0805e-15, 4.2004e-15, 3.3222e-16, 9.3708e-16,\n 2.4378e-16, 1.1232e-16, 2.5260e-15, 1.0380e-17, 2.6614e-15, 3.0371e-15,\n 6.8396e-16, 2.8456e-17, 2.8393e-17, 1.7534e-15, 1.2805e-15, 4.0250e-17,\n 7.8176e-15, 1.6593e-16, 9.4364e-16, 1.4143e-15, 6.1571e-16, 4.2558e-16,\n 1.5001e-17, 5.1891e-15, 2.6842e-15, 2.8152e-17, 3.3649e-15, 9.5472e-17,\n 4.2229e-15, 4.2218e-15, 2.2218e-15, 1.9563e-15, 1.9718e-15, 6.6727e-18,\n 1.2842e-15, 3.5237e-17, 3.1982e-17, 5.3380e-17, 1.3936e-16, 2.2597e-17,\n 4.8849e-15, 1.3201e-15, 2.5522e-15, 5.4997e-16, 3.0235e-15, 7.9115e-16,\n 2.4313e-15, 2.2289e-17, 6.6297e-16, 4.1573e-15, 1.1872e-15, 3.2451e-15,\n 2.2015e-15, 8.0504e-17, 2.0826e-15, 1.5351e-15, 5.6070e-16, 7.7485e-16,\n 3.4036e-16, 1.6933e-15, 2.1821e-17, 1.1192e-15, 6.5696e-16, 2.9340e-16,\n 6.8409e-15, 9.5866e-17, 3.6297e-15, 8.3192e-16, 2.3831e-16, 2.3582e-15,\n 1.8363e-15, 1.6531e-15, 2.4631e-17, 1.1872e-15, 2.8648e-16, 1.5762e-15,\n 2.7511e-16, 3.2566e-17, 1.4388e-16, 2.0053e-17, 4.7188e-16, 2.8812e-15,\n 1.5324e-15, 2.8706e-15, 1.7013e-15, 5.8396e-17, 6.5878e-16, 3.6249e-15,\n 1.2219e-15, 2.5019e-16, 4.6233e-16, 1.1148e-15, 7.3459e-18, 4.7887e-16,\n 1.0807e-15, 2.4401e-17, 1.1036e-15, 1.9374e-15, 6.0079e-15, 1.8816e-16,\n 2.0436e-15, 8.0562e-16, 1.2003e-15, 3.0349e-15, 7.3403e-15, 1.7840e-15,\n 1.6762e-15, 1.0389e-16, 1.8496e-15, 2.1475e-15, 1.9241e-15, 1.1131e-15,\n 1.0093e-15, 2.4501e-15, 2.6546e-16, 5.6754e-15, 1.6387e-17, 9.8884e-16,\n 5.4882e-15, 1.9167e-15, 1.8634e-18, 1.1096e-15, 1.3287e-15, 8.3568e-16,\n 1.4690e-15, 1.6728e-15, 1.3617e-17, 3.3503e-16, 4.0869e-17, 3.2656e-17,\n 1.6110e-15, 2.3833e-17, 3.1664e-17, 1.1227e-15, 3.1891e-17, 2.6458e-15,\n 1.4539e-14, 1.5255e-15, 3.5709e-15, 4.5726e-16, 9.8404e-15, 2.6911e-15,\n 8.4386e-16, 4.8056e-16, 1.0043e-15, 3.3837e-15, 1.0464e-15, 3.4499e-15,\n 2.9915e-15, 2.8804e-15, 2.2536e-15, 1.1913e-16, 5.1959e-15, 8.0774e-16,\n 7.4263e-16, 2.8600e-17, 2.5932e-15, 1.0519e-17, 3.8508e-16, 5.5268e-15,\n 1.5664e-15, 5.1914e-15, 3.2941e-16, 2.3877e-15, 1.7851e-15, 4.5861e-16,\n 5.7436e-16, 3.5690e-17, 2.2825e-15, 3.2928e-17, 7.7847e-16, 9.9113e-16,\n 3.0912e-15, 3.3333e-16, 1.0146e-15, 3.5104e-16, 5.3805e-16, 5.3992e-16,\n 5.9907e-16, 1.1555e-15, 2.5136e-16, 1.8493e-16, 1.3267e-17, 3.9835e-16,\n 2.5102e-15, 5.6149e-15, 1.0400e-15, 4.3128e-15, 1.2759e-15, 2.8609e-16,\n 1.8384e-16, 5.0161e-16, 1.6326e-15, 5.6104e-15, 3.1687e-16, 5.7621e-17,\n 1.6861e-15, 1.2519e-16, 3.7848e-15, 2.1732e-15, 5.9284e-15, 3.7612e-15,\n 3.1824e-16, 1.3414e-18, 2.4580e-15, 1.6636e-15, 5.3544e-16, 1.3472e-17,\n 6.8708e-15, 1.2970e-17, 3.8725e-15, 6.5153e-16, 1.0746e-15, 2.4095e-15,\n 6.7452e-16, 2.0479e-15, 7.1411e-16, 2.4686e-16, 3.4292e-15, 5.6105e-15,\n 2.1816e-17, 1.8673e-17, 1.7553e-15, 1.5727e-15, 1.2279e-16, 3.9914e-15,\n 1.7296e-15, 2.1962e-16, 6.2571e-16, 4.0234e-18, 3.9753e-16, 5.4281e-16,\n 1.0772e-15, 3.2616e-16, 3.7536e-15, 5.1338e-15, 2.2731e-15, 1.2851e-15,\n 1.3060e-15, 2.4870e-15, 3.7216e-16, 4.6663e-17, 1.7998e-16, 5.1119e-16,\n 1.2287e-15, 2.0509e-17, 1.3909e-14, 2.1648e-15, 4.8533e-18, 2.5078e-15,\n 3.1682e-15, 7.3179e-15, 3.4435e-15, 4.6561e-15, 4.6575e-17, 4.5069e-17,\n 1.6334e-15, 2.4014e-15, 3.5434e-15, 9.9778e-16, 4.4009e-16, 5.8993e-16,\n 4.8196e-18, 3.2411e-15, 5.0648e-16, 3.0750e-15, 1.0321e-15, 2.5070e-15,\n 6.7712e-15, 3.8552e-15, 1.7736e-15, 1.5204e-15, 2.0216e-15, 3.3610e-17,\n 1.2556e-16, 4.1253e-15, 1.5866e-15, 4.0104e-15, 4.4842e-17, 8.0379e-16,\n 1.2652e-15, 3.5296e-15, 3.2515e-15, 1.0008e-15, 2.2633e-15, 4.6789e-15,\n 2.5890e-16, 1.9362e-17, 6.4699e-16, 1.0778e-15, 4.9891e-15, 4.0635e-17,\n 6.2479e-15, 4.4919e-16, 8.5067e-16, 3.1049e-16, 5.2348e-15, 1.6748e-17,\n 6.8805e-15, 1.8152e-16, 1.7980e-16, 1.1903e-15, 1.1932e-15, 4.7603e-18,\n 5.1499e-18, 3.2207e-16, 1.7682e-15, 1.1198e-15, 7.2375e-16, 7.7195e-15,\n 1.1482e-16, 5.5625e-15, 1.5790e-15, 1.1619e-15, 4.2855e-15, 2.0207e-15,\n 3.5463e-15, 1.0954e-15, 6.8845e-16, 2.7572e-15, 1.4330e-17, 8.6300e-16,\n 3.6032e-17, 2.5601e-16, 2.9862e-15, 1.6598e-16, 9.9629e-16, 2.4410e-15,\n 2.3510e-15, 7.5247e-17, 1.6801e-17, 2.6123e-15, 1.4939e-17, 1.8483e-17,\n 1.4284e-16, 2.7234e-15, 3.1499e-15, 7.1110e-17, 1.9577e-16, 2.5682e-15,\n 5.9784e-15, 4.3690e-15, 1.1450e-16, 3.1845e-15, 6.0608e-16, 6.4615e-17,\n 4.5164e-15, 5.9337e-17, 2.3556e-17, 1.3845e-15, 1.8151e-16, 3.4498e-15,\n 3.7044e-16, 7.2302e-16, 1.1518e-16, 7.7455e-16, 1.7326e-15, 3.2245e-15,\n 1.1944e-17, 8.1411e-15, 1.7148e-15, 3.5402e-17, 7.3992e-16, 4.5240e-16,\n 5.2098e-17, 2.8327e-15, 3.4173e-15, 9.9103e-16, 7.4237e-16, 3.7331e-15,\n 6.8580e-16, 4.0136e-15, 1.1194e-15, 1.9046e-15, 2.1320e-16, 3.1845e-15,\n 1.6620e-15, 1.1748e-16, 7.9849e-17, 2.1594e-15, 5.1115e-15, 7.3225e-16,\n 2.2494e-15, 2.7732e-15, 1.3980e-15, 3.0110e-15, 1.2763e-15, 4.2516e-15,\n 4.8614e-15, 1.1320e-15, 6.9516e-16, 9.8067e-17, 4.6297e-17, 1.3054e-15,\n 1.1053e-16, 1.4206e-14, 2.0919e-16, 4.3043e-16, 2.7838e-15, 3.5631e-15,\n 4.4415e-16, 2.5324e-15, 7.5976e-16, 5.6775e-15, 4.5819e-15, 3.2258e-16,\n 1.1225e-15, 1.1221e-15, 1.0149e-14, 3.6697e-17, 2.0579e-15, 5.3138e-16,\n 2.8009e-17, 1.1466e-15, 2.6297e-15, 7.2539e-17, 5.2432e-15, 7.5456e-16,\n 8.3853e-16, 5.0069e-15, 6.1598e-17, 1.6526e-15, 8.9263e-17, 3.0073e-15,\n 1.7172e-15, 7.4151e-16, 6.5926e-16, 1.5359e-15, 1.6911e-15, 1.8682e-15,\n 8.9187e-16, 3.5977e-17, 2.5614e-16, 4.2176e-16, 5.4594e-15, 3.9795e-16,\n 5.2239e-15, 8.3895e-18, 6.4229e-16, 5.6030e-16, 1.0205e-16, 8.2968e-17,\n 2.6059e-16, 7.6625e-15, 2.2891e-15, 4.8729e-16, 1.0295e-14, 4.3385e-15,\n 1.7821e-15, 1.2274e-15, 1.6417e-15, 1.8129e-15, 1.1509e-14, 3.4520e-15,\n 1.7958e-16, 4.4189e-15, 1.2576e-15, 2.0023e-15, 4.8382e-15, 3.7067e-15,\n 3.3081e-15, 5.8437e-17, 1.2312e-16, 2.1667e-15, 2.2718e-15, 5.5971e-16,\n 4.6418e-17, 3.1214e-16, 4.4833e-15, 2.2392e-17, 3.9037e-17, 6.3530e-16,\n 1.6377e-16, 2.4810e-16, 2.2175e-15, 1.0138e-14, 3.6587e-15, 6.5753e-15,\n 5.4167e-16, 3.4505e-15, 3.1686e-15, 1.8858e-15, 6.0405e-16, 2.6171e-17,\n 3.3022e-15, 8.9890e-18, 2.2128e-17, 1.3300e-17, 5.3623e-16, 3.4848e-15,\n 2.5852e-17, 4.7833e-16, 1.6945e-16, 7.1709e-16, 1.0987e-15, 5.1042e-15,\n 3.4603e-16, 1.3773e-15, 1.8478e-15, 2.3113e-17, 3.3444e-17, 2.3561e-15,\n 4.4724e-15, 1.3667e-15, 2.2804e-16, 5.8098e-16, 7.7674e-16, 1.4604e-15,\n 7.0045e-16, 1.5200e-15, 1.2933e-16, 4.0059e-15, 2.3867e-15, 9.4602e-16,\n 3.1027e-15, 8.8431e-16, 3.0389e-17, 5.7733e-16, 4.3543e-15, 2.8537e-16,\n 9.6019e-18, 4.3199e-15, 1.0191e-14, 3.1725e-15, 6.8299e-16, 1.3277e-15,\n 1.3460e-16, 3.2156e-15, 1.2356e-15, 5.7475e-16, 1.1339e-15, 3.1057e-15,\n 8.5181e-16, 7.6268e-16, 8.8325e-15, 6.5135e-16, 2.0504e-15, 8.7704e-17,\n 2.8968e-16, 3.1877e-16, 4.4361e-16, 1.0233e-15, 3.0566e-16, 1.9982e-15,\n 8.7968e-17, 1.7746e-15, 3.4431e-17, 5.9695e-17, 5.0572e-16, 3.0839e-15,\n 7.8493e-15, 4.4489e-16, 3.5277e-15, 2.2182e-15, 2.9104e-17, 5.5724e-16,\n 2.3964e-16, 3.3691e-16, 2.4627e-15, 3.8578e-17, 2.0485e-18, 5.4220e-16,\n 1.0334e-17, 6.5913e-16, 1.1420e-15, 8.4665e-16, 1.5830e-14, 2.0442e-16,\n 3.5250e-17, 7.3776e-16, 2.7130e-16, 5.4877e-16, 3.2559e-15, 2.7330e-15,\n 2.1748e-15, 5.7255e-17, 1.6757e-15, 1.0022e-16, 2.6489e-15, 1.7267e-15,\n 4.4763e-15, 1.0454e-17, 1.4374e-15, 2.0784e-15, 3.0785e-16, 8.2362e-15,\n 1.1824e-15, 4.9288e-16, 2.2226e-15, 1.4668e-16, 1.0104e-15, 4.7047e-15,\n 3.6103e-15, 1.5387e-15, 5.5354e-16, 8.1494e-16, 4.3139e-15, 1.5519e-15,\n 2.7044e-16, 3.6060e-15, 2.4196e-16, 8.4355e-17], device='cuda:0')" + }, + "34": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -1.0361e-15, 7.8923e-17, ..., 5.4785e-15,\n 3.4910e-15, 7.7894e-17],\n [ 5.6052e-45, -9.7901e-17, 1.2858e-17, ..., -6.0132e-16,\n -1.8707e-15, 9.6582e-18],\n [-5.6052e-45, 7.9502e-16, 1.3092e-16, ..., 5.2573e-15,\n 9.0774e-15, 9.5546e-17],\n ...,\n [-5.6052e-45, 6.8038e-16, -6.6214e-18, ..., -4.3571e-15,\n -3.5652e-15, -8.4575e-17],\n [-5.6052e-45, 1.1363e-16, 7.9367e-18, ..., 3.0038e-16,\n 1.6470e-16, 2.6270e-18],\n [ 5.6052e-45, -1.7385e-16, -8.0833e-19, ..., -1.9035e-15,\n -4.8704e-15, -7.9130e-18]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.6876e-15, 5.1922e-15, 3.7050e-15, ..., 1.0557e-17, 2.4128e-16,\n 2.9119e-15],\n [1.0561e-15, 1.4894e-15, 1.7183e-15, ..., 8.8002e-19, 8.0925e-17,\n 1.6389e-15],\n [5.6253e-16, 2.0479e-15, 2.0440e-15, ..., 4.6076e-18, 1.4216e-16,\n 1.0468e-15],\n ...,\n [7.4641e-16, 8.4839e-16, 6.6644e-16, ..., 1.8730e-17, 1.6107e-16,\n 7.9805e-16],\n [2.5221e-17, 5.0806e-17, 7.5944e-17, ..., 4.3686e-19, 3.8531e-18,\n 3.6247e-17],\n [4.0443e-17, 6.2740e-17, 7.1677e-17, ..., 1.1714e-19, 1.1727e-17,\n 5.5626e-17]], device='cuda:0')" + }, + "35": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.3836e-11, -1.2885e-12, 1.3711e-12, 1.1666e-12, 2.9560e-12,\n -2.2265e-12, -7.1701e-12, 6.8619e-13, -6.5558e-13, 4.3804e-13,\n -2.3485e-12, -1.4055e-12, -2.2978e-13, -1.1725e-11, 1.2384e-13,\n -3.2399e-14, 6.5168e-12, -2.2360e-13, 1.3592e-13, -4.9916e-12,\n -4.2307e-13, 4.9238e-12, -1.4625e-12, -2.6016e-14, 8.8137e-12,\n -1.3726e-12, -3.0526e-13, -2.1524e-12, -4.2972e-13, 1.8597e-12,\n -4.9696e-13, -1.2766e-12, 4.5195e-12, 6.9352e-12, 8.3271e-13,\n -2.9246e-12, 2.0338e-12, 4.5774e-12, -2.2214e-13, 3.4935e-12,\n -7.0526e-12, 1.7763e-13, -6.6871e-13, -7.8360e-12, -1.2341e-12,\n -4.7063e-12, 1.3989e-12, 8.6349e-13, -2.0168e-12, -1.7369e-12,\n -1.6536e-12, -1.5431e-12, 7.8124e-12, -1.2447e-12, 1.4190e-12,\n -2.6111e-12, 4.2591e-12, 1.3301e-11, 2.1378e-13, -3.2784e-13,\n -5.9633e-12, 2.5964e-12, 1.2663e-12, -2.4456e-12, -1.5212e-12,\n -1.2166e-12, -9.4361e-13, 6.0456e-12, -1.3386e-12, 7.5727e-12,\n -1.2958e-12, -1.6091e-12, -1.0294e-13, -1.9439e-12, 4.7960e-12,\n 1.6938e-12, 1.4263e-12, 1.9206e-12, 1.5246e-12, -5.9369e-12,\n 6.2257e-12, 6.6789e-12, -1.9306e-13, -5.1777e-13, 1.1412e-11,\n -7.0536e-12, -1.0961e-12, -1.5275e-13, -3.1499e-12, -1.3202e-12,\n 2.9403e-12, 3.1876e-12, -1.0787e-12, -2.5022e-12, -2.1342e-12,\n -3.8805e-12, -3.2222e-12, -6.7691e-12, -2.3183e-12, -4.2975e-12,\n -4.0256e-12, -1.8260e-12, 1.7282e-12, -1.3638e-12, -7.5823e-13,\n -8.5872e-13, 2.7683e-12, 1.4680e-12, 3.2728e-13, -1.4144e-13,\n 3.6417e-12, -5.6397e-13, 2.4930e-12, -2.4049e-12, 4.4063e-12,\n -8.0128e-12, 6.5223e-17, 6.9442e-12, -1.0325e-12, 1.3678e-12,\n 1.6922e-12, -9.5109e-12, -3.7346e-13, -3.8913e-12, 6.9877e-12,\n -1.2677e-11, -9.9085e-13, -3.9138e-12, -9.4726e-13, -2.7076e-12,\n -1.4377e-12, -5.2421e-12, 3.6127e-12, -2.0350e-12, -1.4613e-12,\n 1.2477e-12, -1.5210e-12, -7.0560e-12, -1.6260e-12, 1.5279e-12,\n -1.0256e-12, -9.5810e-13, -9.2715e-12, -1.4755e-12, 2.7380e-12,\n 3.4369e-12, 5.7196e-12, -1.5072e-12, 9.6078e-12, -1.8541e-12,\n -5.2804e-13, -1.4516e-12, 1.2007e-11, 8.9458e-12, 2.8704e-12,\n -2.6131e-13, 4.2289e-12, 2.1423e-12, -1.5905e-12, -1.2191e-12,\n 7.5600e-13, 2.7302e-13, -4.1852e-13, 2.7323e-12, -1.2663e-14,\n -2.1373e-12, -6.6306e-13, -3.2284e-13, 6.5076e-12, 3.7649e-14,\n 5.2200e-12, 2.1482e-12, -7.0919e-12, 4.8444e-12, 1.3278e-11,\n -2.2387e-12, 1.4072e-12, -2.5173e-12, -4.0223e-12, -1.5582e-12,\n 4.9333e-12, 7.9014e-12, 1.6869e-12, 8.3855e-12, -3.5408e-12,\n -1.0823e-12, -4.7077e-12, -8.0577e-13, -3.9543e-12, -4.4718e-12,\n -6.6506e-13, -5.7944e-12, -2.7200e-12, -1.5920e-12, -6.9139e-12,\n 7.2019e-12, -2.8871e-12, -1.5454e-14, -2.0286e-12, 8.0985e-13,\n 1.3233e-12, 1.4488e-12, -2.9148e-12, -2.7012e-12, -3.1902e-12,\n -1.2333e-11, -1.4452e-12, -7.4440e-12, -2.4073e-12, -4.6013e-12,\n 1.3392e-12, 6.5391e-12, 1.5299e-12, -1.3007e-12, -8.9765e-13,\n 4.3333e-12, -2.5958e-13, -1.9724e-12, -1.8159e-12, 8.3815e-13,\n -3.7989e-12, 7.6076e-12, -1.1716e-12, -6.0229e-12, -2.0910e-12,\n 1.7802e-12, -1.1134e-12, -1.5749e-12, 7.5212e-12, -4.2730e-14,\n -1.5083e-12, 4.8315e-12, -1.8485e-12, 4.6843e-13, 4.4785e-12,\n -1.8288e-12, -2.0137e-12, -1.1848e-11, 4.2163e-13, -1.3839e-12,\n -1.1860e-12, -5.3205e-12, -4.9036e-13, -9.1948e-12, -2.1477e-12,\n -1.8907e-12, 1.0760e-12, -7.8053e-13, 2.3066e-12, 2.6630e-12,\n -2.5685e-12, -1.0797e-12, -1.8677e-12, -2.7667e-13, -1.6279e-12,\n 5.1182e-13, -4.2383e-12, -1.4166e-12, -6.2868e-12, -1.9239e-12,\n 6.7983e-12, -2.4725e-12, 2.8752e-12, -6.2759e-13, -2.4873e-12,\n -6.0951e-12, -1.4203e-12, -9.4442e-13, -6.0379e-13, -1.0097e-12,\n 4.3014e-12, 3.7729e-12, -8.9068e-12, -1.0451e-12, -1.6965e-13,\n 2.4292e-13, 1.3548e-13, -6.2487e-13, -1.0905e-11, -1.8914e-12,\n 3.5132e-12, -2.3625e-12, -1.5539e-12, -4.9075e-12, -3.3487e-12,\n 8.7556e-12, -3.5373e-12, -2.1695e-12, 3.4798e-12, -5.3509e-13,\n 6.0277e-13, -5.9488e-12, -1.0611e-12, 1.0139e-12, -3.6913e-12,\n -2.2004e-12, 7.6836e-13, -7.8484e-12, -8.5780e-12, 1.1622e-12,\n -9.6588e-12, -3.8238e-12, -1.1211e-11, -9.5168e-13, -1.2955e-12,\n -1.4551e-12, -1.4939e-12, -4.2998e-13, -2.3981e-12, 2.0572e-12,\n -4.1739e-12, -1.6588e-12, 5.9688e-14, 8.0766e-12, -2.5480e-12,\n -2.6737e-13, -3.9259e-12, 1.1243e-11, -8.7382e-12, 1.1607e-11,\n -2.9853e-12, -2.4069e-12, -3.7582e-12, -5.6566e-12, -9.7690e-13,\n -9.4463e-13, -1.8745e-12, 3.0766e-13, -5.9922e-12, 1.0753e-11,\n -2.1888e-12, 1.4577e-12, 1.3766e-12, -1.6123e-12, 2.2416e-11,\n 1.1516e-12, -1.6230e-12, 1.9089e-12, 1.6492e-11, -5.8602e-15,\n -1.7591e-12, 4.4122e-12, -1.4179e-12, -5.9852e-12, -3.5727e-12,\n -2.6057e-12, -1.3596e-12, 9.4815e-12, -3.9758e-12, 1.1146e-11,\n -3.7812e-13, 2.0640e-12, 2.9632e-12, -7.9671e-13, -1.1963e-12,\n -1.2724e-12, -4.0543e-12, -6.3272e-12, 6.7524e-12, -2.8154e-12,\n -1.6733e-12, -7.7331e-13, -1.2501e-12, -1.0790e-11, 2.4081e-12,\n 7.0033e-12, -7.5276e-13, -3.4479e-12, -2.1919e-12, -6.9097e-13,\n -7.8658e-13, 1.1267e-11, 3.3957e-12, -1.3719e-12, -1.9579e-12,\n -1.9168e-12, -1.1973e-13, -1.9559e-12, -3.1898e-12, -3.3632e-12,\n -6.2945e-13, 8.8764e-12, -1.3079e-12, -3.8099e-12, 2.2242e-12,\n 1.5873e-12, -7.5902e-13, -1.1903e-12, 5.3849e-12, 9.4356e-12,\n 8.0252e-12, 5.5101e-14, -1.3035e-12, 4.6377e-12, -1.0751e-12,\n 7.6874e-13, 2.6786e-12, 8.6615e-14, 5.1267e-13, 6.5230e-12,\n 2.3922e-13, 7.2112e-13, -3.2617e-12, -2.5173e-12, -2.6103e-12,\n -1.7330e-12, -1.2968e-12, -2.5454e-12, -4.0054e-12, 3.3907e-12,\n 1.6370e-13, 1.2506e-11, -6.5587e-13, -3.2738e-12, -3.0926e-12,\n -1.9563e-12, 1.0074e-11, 4.7194e-12, 2.7387e-12, -2.3678e-12,\n -2.9110e-12, -1.3945e-12, 7.9416e-13, 3.7668e-13, 2.5145e-12,\n 7.7934e-14, 2.8222e-12, -1.3896e-12, 8.0071e-12, -1.5482e-11,\n -1.6616e-12, 1.0133e-12, 6.6246e-13, 7.3893e-12, 5.2381e-13,\n 6.0825e-12, -1.1866e-12, -6.0204e-13, 7.4476e-13, -5.2929e-12,\n -5.3254e-12, -6.3432e-13, 1.7966e-12, 5.6282e-14, 8.2830e-12,\n -4.6784e-13, -9.7458e-12, -1.7206e-12, -2.6727e-12, -1.1459e-12,\n -3.8681e-13, -1.5178e-12, -3.8201e-12, 7.1841e-13, -3.7034e-12,\n 2.3744e-12, -2.2519e-12, -1.0836e-12, -1.2262e-12, -1.1052e-12,\n 3.0655e-12, 4.7718e-12, -3.4189e-12, -1.8602e-13, 3.6259e-12,\n -8.0241e-12, -1.0280e-12, 1.4556e-12, -5.8109e-12, -6.0060e-12,\n -2.1686e-12, 4.2080e-12, -1.6264e-12, -1.6515e-13, 7.1290e-13,\n -5.3459e-13, -7.7099e-13, 5.7610e-12, 2.8912e-13, -1.0731e-12,\n -2.1296e-12, -1.4004e-12, 8.4267e-12, 2.5360e-13, 8.9910e-12,\n 2.9160e-12, 1.7002e-11, 1.6813e-12, -1.2384e-12, -3.2926e-12,\n -2.6122e-12, 1.3425e-11, -5.1953e-12, -1.1676e-12, -1.3904e-12,\n 4.0973e-12, -6.3591e-12, -5.2551e-12, -1.1203e-11, -4.0467e-12,\n -2.3956e-12, 9.8985e-13, 4.7407e-12, 1.7822e-12, -4.8260e-12,\n -3.1122e-12, -3.5416e-12, -9.2813e-13, 2.6152e-12, -1.1956e-12,\n -6.0317e-12, 1.0512e-12, -1.2273e-12, -1.4411e-11, 2.3585e-12,\n -2.6837e-12, 7.2827e-12, 3.5733e-12, -2.8708e-12, 3.2714e-12,\n 1.9171e-13, 2.4426e-12, -6.4262e-12, -3.6159e-12, 1.7706e-12,\n 2.3045e-12, -3.8552e-12, 2.0703e-12, -8.7797e-12, -1.2687e-12,\n -8.0862e-13, -4.0109e-12, -2.4600e-12, 2.7216e-11, 2.7686e-12,\n 5.9381e-12, 5.4773e-12, 4.0887e-12, 2.6699e-12, -1.1124e-12,\n -7.7653e-13, -3.7018e-12, -1.1416e-12, 8.1253e-12, -2.1331e-13,\n -3.0250e-12, 9.3969e-12, 2.4903e-12, -2.4130e-12, -3.2853e-12,\n 1.4401e-12, -9.7966e-13, 1.8474e-12, 9.7310e-13, -3.2871e-13,\n -8.9142e-13, -1.4078e-12, 5.0690e-12, 2.7090e-12, 2.2373e-12,\n 1.8517e-11, -1.6044e-12, 4.7711e-12, 1.0610e-12, -2.3227e-12,\n 2.1406e-12, -2.6602e-13, -1.2240e-12, 1.8486e-12, -2.9567e-12,\n 5.8316e-12, 1.0176e-11, 1.4360e-11, -1.3187e-12, -1.3567e-13,\n -1.2334e-12, -1.8257e-12, 4.5282e-12, 1.4751e-12, -8.5388e-14,\n 2.3715e-12, -2.0177e-12, 7.9467e-12, 2.6490e-12, 5.2052e-12,\n -1.0198e-12, -9.6696e-13, -1.0555e-12, -1.1888e-11, 4.4337e-12,\n -1.7063e-12, -1.4215e-12, 8.7153e-12, -2.3191e-12, -6.3588e-12,\n 5.4839e-12, -4.3178e-12, -1.2028e-12, -9.6061e-13, -4.9282e-12,\n -8.0568e-13, -8.8284e-13, -1.0789e-12, 3.3194e-12, -1.5156e-13,\n -1.2648e-12, -9.9693e-12, 7.4366e-12, 1.8697e-12, -7.1314e-14,\n 1.4067e-12, -3.1027e-12, 8.7874e-15, 3.0669e-12, -4.2260e-12,\n -1.2864e-12, -1.5794e-12, -4.0605e-12, -2.8879e-12, -4.8306e-13,\n -2.9402e-12, 3.9086e-12, -1.0863e-12, -2.8618e-12, -2.8488e-12,\n 1.5565e-12, 3.8040e-12, -2.4966e-12, 1.1832e-11, 2.0205e-12,\n -1.4380e-12, 2.7325e-12, -4.0876e-12, 5.8314e-12, -2.1154e-12,\n 6.9457e-12, 9.0419e-12, -5.6069e-12, 1.7375e-12, 3.2830e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([4.4287e-12, 1.1926e-12, 2.7058e-12, 5.4729e-12, 1.6112e-13, 2.2564e-12,\n 2.1151e-13, 2.4080e-13, 2.5423e-13, 3.6176e-12, 8.0544e-12, 2.5244e-13,\n 3.9899e-12, 6.8026e-14, 9.7941e-13, 1.6122e-12, 1.3418e-12, 1.1377e-12,\n 2.8337e-12, 3.0678e-13, 4.5774e-14, 2.5048e-12, 2.6535e-12, 4.1149e-12,\n 4.9908e-12, 8.6343e-14, 1.1160e-12, 1.7201e-12, 4.0519e-12, 6.2689e-13,\n 3.3853e-12, 9.7478e-13, 1.9333e-13, 2.3337e-12, 1.6191e-12, 2.6133e-12,\n 1.2591e-13, 5.9327e-12, 2.0241e-12, 4.3115e-12, 1.1075e-13, 1.4331e-12,\n 6.6632e-13, 3.5411e-13, 4.4504e-14, 1.4223e-12, 7.2035e-12, 6.8472e-12,\n 2.2726e-12, 1.5431e-12, 3.9851e-14, 2.4719e-12, 1.8151e-12, 2.5249e-12,\n 9.2716e-13, 1.5592e-12, 1.0277e-13, 2.4103e-12, 5.5538e-12, 8.6952e-14,\n 2.0691e-12, 9.6701e-14, 3.3967e-12, 5.3483e-13, 1.4984e-13, 5.7169e-13,\n 2.4363e-12, 4.1272e-12, 3.0264e-14, 2.4350e-12, 1.0845e-13, 8.1194e-14,\n 6.2935e-12, 2.5777e-12, 8.3007e-12, 7.4057e-13, 1.3861e-13, 4.3753e-14,\n 5.6992e-14, 4.5002e-12, 6.1791e-13, 2.1121e-12, 3.1115e-13, 1.0762e-12,\n 6.8915e-12, 5.0029e-12, 2.5192e-13, 9.2690e-12, 1.5831e-12, 1.6535e-12,\n 3.0937e-12, 2.0665e-12, 1.4018e-13, 4.6260e-13, 1.5672e-12, 3.0818e-14,\n 2.8724e-13, 1.0631e-12, 1.7563e-13, 2.9795e-12, 1.4234e-13, 4.2539e-13,\n 4.3314e-12, 7.4463e-14, 1.7946e-12, 1.9612e-13, 1.8659e-13, 4.1696e-13,\n 8.6219e-13, 1.9095e-13, 5.2985e-12, 3.9716e-14, 4.3783e-13, 1.1864e-12,\n 4.5975e-14, 9.1443e-13, 1.6809e-12, 5.8981e-12, 1.4496e-13, 7.9044e-12,\n 4.3797e-12, 7.7990e-13, 4.3609e-14, 2.5127e-12, 7.4579e-14, 6.7019e-12,\n 5.4144e-13, 4.2003e-13, 2.6447e-12, 4.4263e-12, 1.2764e-13, 9.3731e-13,\n 6.4993e-12, 1.1010e-13, 8.9271e-13, 2.0419e-12, 1.8539e-12, 1.9664e-12,\n 1.4881e-12, 5.0223e-13, 2.4632e-12, 1.3450e-12, 1.6183e-12, 2.2338e-12,\n 9.3049e-12, 5.7545e-12, 6.9065e-14, 4.1552e-12, 1.6647e-12, 7.3021e-13,\n 3.8294e-13, 1.9015e-12, 1.4856e-12, 7.7103e-12, 1.0328e-12, 7.4193e-14,\n 7.5222e-13, 3.7642e-13, 2.1747e-12, 9.8773e-14, 1.0895e-12, 1.1507e-12,\n 2.1033e-14, 2.9578e-12, 1.1001e-11, 2.4369e-12, 4.2357e-13, 3.1218e-13,\n 1.1678e-11, 3.0187e-12, 5.4055e-12, 1.6264e-12, 5.3807e-12, 2.5886e-13,\n 3.0447e-11, 6.4600e-12, 7.3988e-13, 1.5210e-12, 3.2220e-12, 2.4314e-12,\n 2.0731e-13, 6.4863e-12, 3.3259e-12, 5.2922e-12, 4.1979e-12, 1.6769e-11,\n 1.0386e-12, 2.2642e-12, 7.8903e-13, 1.3599e-12, 8.1249e-14, 2.9986e-12,\n 2.4713e-12, 1.2466e-13, 2.6828e-12, 1.4493e-13, 1.3315e-12, 5.5539e-13,\n 4.3878e-12, 2.4984e-13, 1.4495e-12, 4.1719e-12, 1.1225e-12, 6.5666e-14,\n 1.6403e-13, 4.3948e-12, 1.1900e-13, 1.8252e-12, 1.1442e-13, 1.6856e-12,\n 4.4822e-13, 3.7147e-12, 4.0406e-13, 1.0383e-13, 6.5949e-13, 4.5485e-12,\n 1.7563e-12, 1.3534e-13, 2.2827e-12, 2.8732e-13, 2.7326e-13, 2.5415e-12,\n 7.8073e-14, 7.8712e-12, 6.7577e-13, 1.3174e-12, 1.6062e-12, 1.5230e-13,\n 2.0255e-13, 1.3972e-12, 4.8851e-14, 2.6968e-13, 1.9743e-12, 3.4505e-13,\n 1.0460e-13, 6.1416e-14, 3.8800e-13, 2.1039e-12, 5.3398e-12, 2.9305e-13,\n 4.1076e-13, 9.6715e-13, 9.7927e-12, 2.0679e-12, 8.9569e-13, 1.6573e-13,\n 1.5036e-13, 3.2902e-13, 1.0918e-12, 1.8416e-13, 2.4219e-12, 1.3180e-12,\n 1.9540e-12, 7.1704e-12, 3.9576e-14, 1.0754e-13, 6.7457e-13, 1.7919e-13,\n 2.5887e-13, 1.7629e-12, 7.1083e-14, 4.0303e-12, 3.4024e-13, 1.6742e-13,\n 1.1977e-12, 3.1874e-12, 1.8169e-13, 2.8783e-12, 3.1601e-12, 5.3523e-13,\n 2.6687e-13, 1.0846e-12, 5.7501e-13, 3.7842e-12, 1.0640e-12, 8.7697e-13,\n 6.1210e-14, 4.5656e-12, 8.6508e-13, 1.3565e-12, 2.3997e-13, 1.2745e-12,\n 2.6670e-12, 6.3150e-13, 2.3639e-11, 7.9883e-12, 1.9863e-13, 5.0992e-12,\n 1.6150e-13, 1.3195e-13, 1.1230e-12, 2.6982e-12, 1.4979e-13, 1.4658e-12,\n 4.9191e-14, 2.0706e-13, 5.2105e-12, 3.1364e-12, 9.3097e-12, 3.9981e-12,\n 3.1945e-12, 3.0790e-12, 7.5247e-12, 1.1058e-12, 1.3279e-12, 1.1533e-12,\n 1.4005e-12, 4.2264e-12, 2.4054e-12, 6.5858e-14, 7.6811e-12, 4.2744e-13,\n 2.7405e-13, 5.2595e-12, 5.2072e-13, 6.0586e-12, 2.4648e-12, 2.5122e-12,\n 2.1431e-12, 2.6697e-12, 3.1255e-13, 4.2774e-13, 2.9245e-14, 3.8923e-12,\n 3.2455e-13, 2.4979e-13, 2.7287e-13, 2.7351e-12, 9.8567e-14, 2.3280e-13,\n 1.6203e-13, 2.5281e-12, 3.8955e-12, 1.0748e-12, 1.5291e-11, 1.2587e-13,\n 1.6377e-13, 1.2047e-12, 6.0309e-12, 5.8191e-12, 9.6382e-13, 1.7268e-12,\n 8.2630e-13, 3.2329e-12, 1.1606e-12, 3.6096e-13, 2.6590e-14, 8.8349e-13,\n 2.4553e-12, 1.8608e-12, 1.5095e-12, 1.8773e-12, 9.3574e-14, 1.0650e-13,\n 5.8832e-13, 2.3131e-13, 1.0857e-12, 2.8966e-12, 9.8675e-14, 1.2016e-12,\n 8.1093e-13, 3.2442e-13, 4.3052e-13, 2.0082e-12, 5.0021e-13, 1.3961e-12,\n 2.4465e-12, 2.1207e-12, 5.9121e-12, 1.2827e-12, 1.2273e-13, 2.8297e-12,\n 3.1748e-12, 2.2045e-12, 3.4518e-13, 3.6900e-13, 4.3516e-12, 5.5181e-13,\n 8.8299e-12, 3.2048e-12, 1.0872e-12, 2.5759e-12, 3.8169e-14, 4.4548e-12,\n 3.0445e-13, 1.6146e-13, 3.1462e-12, 1.6953e-12, 1.5062e-13, 3.7814e-12,\n 1.1379e-12, 1.6263e-12, 1.4465e-13, 2.0817e-13, 3.1113e-12, 1.1387e-13,\n 1.7887e-13, 1.5991e-13, 1.6234e-12, 5.8408e-12, 1.0370e-12, 1.7149e-12,\n 7.4064e-13, 3.8493e-12, 6.6787e-12, 1.3245e-12, 3.0246e-13, 2.9171e-12,\n 1.6597e-13, 3.9218e-12, 6.2368e-13, 4.2654e-12, 1.2308e-13, 1.2131e-11,\n 7.4841e-13, 1.7247e-12, 2.0002e-12, 3.0516e-12, 2.1306e-12, 2.1591e-12,\n 6.5444e-13, 2.2206e-12, 1.7019e-12, 8.7774e-13, 2.1826e-12, 2.8977e-12,\n 2.5354e-13, 3.6162e-13, 4.2269e-12, 1.6507e-11, 3.0355e-13, 6.1125e-12,\n 1.2866e-13, 1.4679e-11, 8.5792e-13, 6.2275e-12, 2.4012e-12, 4.3099e-14,\n 1.1955e-13, 9.5589e-14, 2.3842e-12, 4.5273e-13, 6.3847e-12, 1.9551e-12,\n 1.2453e-11, 7.0108e-14, 1.9636e-12, 8.4629e-12, 1.1990e-12, 1.5843e-12,\n 6.3832e-14, 1.0481e-13, 2.4652e-12, 9.0484e-13, 4.3784e-12, 3.0371e-12,\n 4.8096e-14, 5.2090e-13, 4.8231e-13, 1.4999e-13, 1.1925e-12, 7.7291e-12,\n 3.4566e-12, 2.5036e-12, 2.4261e-13, 3.5520e-12, 5.5491e-13, 8.8635e-14,\n 3.5048e-12, 4.0263e-12, 8.1306e-13, 1.9358e-13, 5.7221e-12, 4.5156e-14,\n 3.9767e-12, 2.9985e-13, 3.4860e-13, 8.1805e-12, 2.0692e-13, 9.8563e-13,\n 1.2288e-12, 8.1756e-13, 2.7386e-12, 1.4073e-13, 2.4610e-12, 1.0541e-13,\n 9.6948e-12, 1.8788e-12, 1.8045e-13, 6.9814e-12, 1.2261e-11, 2.1846e-11,\n 1.4998e-12, 1.8020e-12, 2.1663e-13, 9.2003e-13, 5.1248e-12, 7.8667e-13,\n 7.0300e-12, 2.1147e-12, 7.0031e-14, 2.2481e-13, 3.5637e-12, 1.4671e-13,\n 1.0158e-12, 5.0133e-12, 5.1964e-13, 1.2966e-12, 9.4317e-13, 7.9980e-13,\n 1.1229e-11, 2.4092e-13, 5.5085e-14, 1.1342e-11, 3.0622e-12, 3.4470e-12,\n 3.9111e-12, 1.0409e-13, 8.7330e-13, 3.8074e-12, 4.4477e-14, 1.0340e-12,\n 1.0685e-12, 3.7855e-12, 3.6010e-12, 1.8318e-13, 1.5313e-13, 2.3437e-12,\n 2.3523e-12, 6.6191e-13, 1.9259e-13, 1.4422e-13, 4.6213e-12, 1.9287e-11,\n 2.7034e-12, 3.2977e-12, 4.3163e-12, 2.9189e-12, 4.5972e-14, 1.8248e-12,\n 5.3569e-13, 8.9794e-13, 1.8136e-12, 1.1585e-12, 1.5936e-12, 1.5884e-12,\n 3.0726e-12, 1.8424e-13, 5.2261e-12, 2.1624e-12, 7.9798e-14, 7.7771e-14,\n 2.2589e-12, 1.1246e-12, 5.4238e-14, 6.8553e-14, 2.6774e-12, 1.1589e-12,\n 1.0498e-11, 4.8858e-12, 1.1879e-11, 1.3169e-12, 8.8974e-13, 3.1389e-12,\n 1.9399e-13, 2.4499e-13, 8.5582e-13, 1.0131e-12, 2.0196e-12, 2.6654e-12,\n 1.4337e-12, 2.4866e-12, 7.7643e-12, 1.0893e-12, 3.0013e-12, 7.5057e-14,\n 6.1567e-13, 1.4851e-13, 3.8991e-12, 1.3763e-13, 5.0355e-13, 4.8945e-12,\n 2.9209e-12, 1.7138e-13, 1.6443e-12, 4.3136e-12, 3.1875e-12, 1.6938e-12,\n 1.3993e-11, 6.8725e-13, 1.0140e-11, 2.1408e-12, 4.3588e-12, 3.8787e-13,\n 3.9432e-12, 3.0320e-12, 3.5354e-13, 3.8333e-13, 1.5664e-13, 3.9744e-12,\n 1.0839e-12, 4.1427e-12, 9.1379e-14, 3.1823e-13, 2.7922e-13, 1.2427e-13,\n 3.1890e-12, 1.5002e-12, 1.6191e-13, 3.0022e-12, 1.9061e-12, 1.4715e-12,\n 2.1450e-12, 4.0289e-12, 2.3735e-12, 8.6901e-14, 4.9737e-12, 1.3582e-12,\n 3.1122e-13, 5.9602e-13, 3.4689e-12, 6.0817e-13, 9.8472e-14, 4.0062e-12,\n 6.0827e-13, 1.4053e-13, 2.1735e-12, 1.0668e-13, 8.7793e-13, 5.1805e-12,\n 1.3501e-13, 4.3852e-13, 3.9741e-13, 8.6164e-13, 4.1766e-13, 1.6969e-12,\n 4.4929e-13, 1.8931e-12, 1.8892e-13, 6.9989e-14], device='cuda:0')" + }, + "36": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 3.2096e-13, 1.1169e-13, 3.2291e-13, 1.0843e-13, 1.0959e-14,\n 1.2644e-14, 2.3254e-15, 1.9646e-14, 3.3358e-14, 1.0543e-14,\n 1.6963e-14, -8.3286e-15, 1.5998e-13, -1.1588e-13, -2.4087e-14,\n 1.5071e-14, 3.7902e-13, 9.7223e-15, -9.3288e-15, -1.3821e-15,\n 3.3611e-15, 3.3702e-13, 2.1896e-17, 3.2707e-14, 2.1328e-13,\n 2.2654e-14, -2.2811e-14, -9.8612e-16, 1.5014e-13, 2.1006e-13,\n -1.8749e-14, -1.0075e-14, 1.4462e-14, 2.5139e-13, 6.1946e-14,\n 8.4499e-15, 2.1406e-13, 1.3603e-13, -2.0862e-14, 1.3874e-13,\n -2.1767e-14, 1.8035e-14, 2.8398e-14, -3.4078e-14, 1.3144e-14,\n 2.6110e-14, 8.8851e-14, 2.4319e-13, 2.0916e-13, -3.4745e-15,\n 1.3874e-14, -7.3006e-15, 1.4828e-13, 8.5015e-14, 1.1866e-13,\n 1.4164e-14, 6.8550e-14, 5.3365e-13, 6.2453e-14, 6.0524e-14,\n -6.9416e-15, 2.5265e-14, -2.5162e-15, -1.8058e-15, 1.1914e-14,\n -8.1320e-15, 5.7593e-14, 8.5646e-14, -2.9388e-15, 2.3334e-13,\n -4.7478e-15, 6.7568e-14, 3.5058e-14, 2.2139e-14, 4.3347e-13,\n 1.6650e-13, 3.8064e-14, 1.2042e-14, 8.7745e-15, -5.8477e-15,\n 2.5510e-13, 5.3939e-13, 3.3009e-14, -2.0406e-14, 3.9398e-13,\n 1.2847e-13, -1.0472e-14, 1.4726e-14, 5.5710e-15, 2.4687e-15,\n 6.0754e-14, 3.7046e-13, 1.1324e-14, 1.1440e-15, -5.5911e-15,\n -2.4408e-15, 8.9715e-14, -1.4511e-14, -1.3408e-15, 1.5263e-13,\n 3.8019e-14, 1.4379e-14, 1.8838e-13, -3.1310e-15, -1.5912e-14,\n -2.0748e-15, 1.9477e-14, -5.3128e-15, -1.9454e-14, 1.7705e-14,\n 3.1385e-13, 6.1897e-14, 1.2789e-13, -1.8300e-15, 2.7601e-14,\n -3.2652e-14, 1.0627e-13, 5.0346e-13, 7.0664e-15, 1.7305e-15,\n -1.7022e-14, -5.9924e-14, 4.2208e-14, 9.8336e-16, 2.9595e-13,\n -1.6210e-13, -1.4724e-14, 3.7414e-15, 9.4330e-14, 4.8883e-15,\n 1.4498e-14, -5.1315e-15, 7.6511e-13, -1.6604e-15, 4.0800e-14,\n -1.9361e-14, 5.3953e-15, 7.3299e-14, 2.1622e-13, 3.2957e-14,\n 3.4449e-16, -8.7088e-15, -4.6303e-14, -1.2673e-15, 4.3961e-13,\n 4.4757e-14, 4.2013e-13, -3.5016e-15, 1.1507e-13, -7.7204e-15,\n -1.0862e-14, 3.3925e-15, 2.8548e-13, 4.9603e-13, 5.8170e-14,\n 5.0048e-14, 1.2769e-13, 4.9301e-14, -5.0011e-15, -8.3943e-15,\n -1.1054e-14, -1.0317e-14, -9.4534e-15, 6.0167e-14, 3.2891e-13,\n 7.5059e-14, 3.0766e-14, 8.8305e-15, 1.7270e-13, -1.1428e-14,\n 2.1856e-13, -1.4797e-14, -2.0278e-14, 1.8674e-14, 8.2346e-14,\n 7.5286e-14, 1.4584e-13, 3.3810e-15, -1.5550e-15, 4.0851e-14,\n 9.4058e-15, 2.3030e-13, -2.9520e-15, 3.3221e-13, 5.3610e-16,\n -2.8220e-15, -2.4649e-15, 8.4426e-14, 5.6917e-15, 7.1774e-14,\n -1.1078e-15, 2.3511e-14, -4.0135e-15, -6.7457e-16, -6.7991e-15,\n -2.5834e-16, 4.3041e-15, 3.2531e-14, 2.6204e-14, -1.2022e-14,\n -1.7821e-14, 2.7181e-13, -6.3724e-15, -8.2401e-16, -2.0938e-15,\n -6.8309e-14, 4.8449e-15, 1.2792e-13, 5.6423e-14, 1.6275e-14,\n -2.1646e-14, 4.7658e-13, 1.2448e-13, 1.6023e-14, -8.2266e-15,\n 5.1760e-14, 2.3252e-13, 8.0758e-14, 5.2116e-13, 4.7804e-14,\n 1.0774e-15, 3.1779e-13, 1.8513e-14, -1.1474e-14, 1.6083e-15,\n 7.8905e-14, -4.6295e-15, -2.4647e-15, 2.2914e-15, 3.2811e-15,\n -5.1811e-15, -2.5363e-15, 2.4033e-16, -2.0960e-14, 1.2294e-15,\n -5.1199e-15, 1.5440e-15, -1.2961e-13, 2.4065e-15, -2.0019e-15,\n -1.7649e-15, -1.0961e-14, 1.1716e-13, -4.0547e-14, -3.3516e-15,\n 1.5827e-14, 2.2874e-15, 8.0663e-15, 5.3251e-14, 6.5476e-14,\n 1.7874e-13, -2.8768e-15, 3.2310e-14, -5.2961e-15, 1.5475e-14,\n -1.4799e-14, -2.3461e-15, 9.6724e-15, -6.0785e-15, -4.6487e-15,\n 1.3563e-16, 1.6502e-13, 7.3520e-14, 2.6002e-14, 1.4486e-15,\n 4.7593e-15, -1.9759e-15, -1.5770e-14, -5.9049e-15, -1.0567e-14,\n 2.0092e-13, 4.3819e-13, -4.5397e-14, -4.7340e-15, -1.2249e-14,\n 1.2053e-14, 1.6312e-15, 1.4522e-15, -1.0485e-13, 2.3670e-15,\n -3.1705e-15, 5.8946e-14, -1.8016e-15, 2.4283e-14, 6.6464e-14,\n 4.5835e-13, 3.5761e-15, 8.7580e-14, -5.3341e-14, 3.0176e-14,\n 4.8101e-15, -8.0476e-15, 8.0455e-15, 1.4918e-14, 1.1040e-16,\n -3.0482e-15, -2.4243e-14, -1.3569e-15, 5.4175e-15, 2.0037e-13,\n -3.9553e-14, 2.7984e-13, -9.6663e-14, 2.7825e-14, 3.8038e-15,\n -5.2338e-15, -2.7047e-15, -8.0138e-15, 3.6173e-17, 2.8415e-16,\n 1.3016e-13, 5.0451e-14, 1.3911e-14, 6.7735e-14, -6.7457e-16,\n -2.3834e-14, 4.9053e-14, 3.0097e-13, -4.9771e-14, 7.7729e-14,\n 6.9744e-16, 1.0504e-13, -9.0137e-16, 2.2635e-14, 1.2035e-15,\n 5.5350e-14, -5.7603e-15, -2.9318e-14, -6.9854e-15, 1.0131e-13,\n 5.4937e-15, -7.5382e-15, 9.7884e-14, -1.3977e-15, 4.6005e-13,\n -7.2632e-15, 2.5399e-14, 2.5420e-15, 1.6043e-12, 5.2043e-14,\n -2.2532e-15, 2.6760e-14, 3.5371e-14, 9.9858e-14, 2.2074e-14,\n 7.4815e-15, -1.7478e-15, 5.9853e-13, 2.9666e-14, 3.4336e-13,\n 1.7439e-13, 6.3741e-14, 1.7214e-14, -1.5043e-14, 6.5659e-15,\n 3.1933e-14, 3.0093e-14, 6.3380e-14, 3.1245e-15, 1.5162e-13,\n -5.3613e-15, 2.3796e-14, -4.0492e-15, -8.9890e-14, -1.4643e-14,\n 1.3807e-13, 5.6947e-14, -1.4024e-15, -3.7397e-15, 3.4221e-14,\n 6.0309e-15, 1.2203e-13, 2.6684e-14, 3.1211e-15, -6.5869e-16,\n -7.9435e-16, 2.5296e-13, 4.8252e-14, 3.9712e-14, 1.4479e-16,\n 5.4820e-15, 1.5302e-13, 5.2767e-14, 3.7166e-14, 5.6861e-14,\n 4.5371e-15, 1.0338e-14, 1.4095e-14, 7.4030e-15, 3.0066e-13,\n 1.3454e-13, 1.6259e-14, -3.4554e-15, 1.2511e-13, -1.1232e-14,\n -3.6277e-14, 2.2558e-14, 3.4380e-14, -1.4027e-14, 8.9340e-13,\n 6.2720e-14, 1.0245e-14, -1.9737e-16, 1.4049e-14, 4.7673e-13,\n 3.6230e-14, -1.2602e-14, 1.2404e-13, 2.2743e-13, 2.5179e-13,\n -2.2561e-14, 4.2090e-13, 1.1630e-14, 9.6224e-17, 1.1097e-14,\n 9.5555e-15, 3.3255e-13, 5.0921e-14, 1.8812e-13, 8.7641e-15,\n 1.1134e-15, 8.0206e-16, 1.6413e-14, -3.0132e-14, -1.0112e-14,\n 1.3354e-14, 5.8439e-14, 3.6132e-15, 3.5498e-13, -2.6917e-13,\n 7.7864e-15, 3.0642e-14, 3.1406e-14, 3.5612e-13, 1.3619e-13,\n 1.9124e-13, 6.9448e-14, -1.8615e-14, 5.8774e-14, -5.1681e-16,\n 1.3985e-14, 2.4674e-13, 2.6685e-13, 1.2213e-13, 3.9889e-13,\n 5.7069e-15, -2.7132e-14, -2.5550e-15, 2.1189e-14, -4.1021e-15,\n 4.0550e-17, -4.0256e-15, 1.3661e-14, 4.0269e-14, 7.7297e-14,\n 3.5074e-13, -1.9383e-15, 1.3335e-14, -7.6657e-15, 3.8499e-15,\n -3.6124e-15, 4.2561e-14, 1.6394e-15, 5.8668e-14, 8.6402e-15,\n -3.8630e-14, 1.0894e-15, -2.4760e-14, 4.8037e-14, -8.7114e-15,\n 5.7937e-14, 2.0579e-13, -6.3556e-15, 6.8338e-14, 1.1746e-13,\n 2.6264e-14, -3.7061e-15, 4.9087e-13, 7.3232e-15, -1.6614e-14,\n -1.7684e-15, -5.6781e-16, 3.3777e-13, -1.6361e-16, 3.1114e-13,\n 2.5043e-14, 9.7586e-13, 4.1609e-15, 1.0525e-14, 1.1023e-13,\n 1.7307e-14, 4.8815e-13, 2.9254e-14, 4.6899e-14, -2.4597e-15,\n 1.4345e-13, 1.8729e-14, 9.1190e-14, -1.1134e-13, 4.3008e-13,\n -8.5460e-16, 4.3701e-14, 1.2611e-13, 1.3168e-14, -4.9353e-17,\n 5.4139e-14, 1.1088e-16, 1.4134e-14, 1.4048e-13, -8.3838e-15,\n 1.2372e-13, 2.7314e-14, 7.8190e-15, -1.4212e-13, 9.6648e-14,\n -4.2204e-15, 3.9071e-13, -9.2325e-15, 4.0320e-14, 1.4783e-13,\n 1.6074e-14, 2.9162e-13, -1.2443e-14, 6.5159e-15, 2.8399e-13,\n 3.4201e-15, -3.5036e-16, 4.3629e-13, -4.8083e-15, 9.0322e-15,\n -6.6793e-15, 3.2729e-14, 4.4221e-15, 1.2836e-12, 1.7983e-14,\n 6.2454e-13, 2.4110e-13, 7.1327e-14, 9.1662e-15, 1.4763e-14,\n 5.6426e-15, 1.0041e-15, 1.3752e-13, 7.1981e-13, -1.3362e-14,\n 9.0080e-14, 7.8203e-13, 3.0476e-14, 1.7240e-13, 2.2547e-15,\n 1.9512e-14, 8.2451e-14, 2.6742e-13, -9.5908e-15, 5.0921e-15,\n 1.6176e-14, 3.1745e-14, 3.3997e-13, 1.5034e-13, -5.3261e-15,\n 6.4130e-13, -9.4525e-15, 7.7058e-14, 1.1525e-13, 2.4387e-15,\n 1.0674e-14, -1.4192e-15, -6.2767e-19, 3.2510e-13, 5.6491e-14,\n 1.3551e-13, 6.2341e-13, 1.9268e-13, -7.5709e-15, 1.1184e-13,\n -8.5442e-17, -2.7178e-16, 1.6685e-14, 9.3964e-14, 5.2500e-14,\n 6.2554e-14, 8.5220e-14, 3.8542e-13, 2.9662e-14, -8.7499e-15,\n 2.6812e-13, 1.3333e-14, 2.5538e-14, -1.3479e-13, 2.4947e-13,\n 1.4794e-15, 1.2831e-16, 5.4068e-13, 2.3615e-15, 4.7476e-14,\n 1.1285e-13, 2.0041e-14, -6.1020e-15, 1.3363e-14, -4.1324e-15,\n 1.3797e-13, 4.2331e-13, 1.3751e-14, 1.5682e-13, 4.2999e-15,\n 3.7505e-14, -5.1978e-14, 7.7609e-13, 3.4178e-14, 7.5217e-14,\n -1.0689e-16, 1.0392e-14, 1.1893e-14, 8.4853e-13, 4.4530e-15,\n 1.1511e-15, -7.1654e-15, 3.8857e-13, -1.3835e-15, -1.2876e-14,\n 1.8385e-13, 1.9917e-13, 4.9511e-14, 2.2987e-14, 4.4405e-16,\n -1.8204e-14, 1.7274e-13, -2.3346e-15, 1.8642e-13, -3.6009e-14,\n 2.8331e-14, -5.3280e-15, 1.2541e-15, 3.4959e-13, -2.9140e-15,\n 1.7466e-13, 8.4417e-13, -1.0396e-14, -3.3540e-14, 7.8757e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([5.9422e-15, 3.7375e-17, 1.0012e-15, 1.7016e-15, 1.7920e-17, 4.3052e-15,\n 4.0542e-17, 1.7488e-17, 3.6276e-17, 1.1257e-15, 3.7332e-15, 5.3145e-18,\n 4.8718e-15, 1.0657e-17, 2.3590e-16, 1.5115e-16, 2.9063e-16, 6.9472e-16,\n 1.0627e-15, 1.3695e-17, 7.9827e-18, 7.2776e-16, 8.3222e-16, 5.6080e-16,\n 1.4764e-15, 4.6858e-17, 1.2258e-17, 1.3831e-15, 5.4599e-15, 4.4830e-17,\n 2.2162e-15, 5.7053e-17, 2.7610e-17, 2.3135e-15, 5.5632e-16, 3.6822e-15,\n 4.9900e-17, 4.2126e-15, 4.0067e-16, 2.3864e-15, 3.8966e-18, 7.8099e-16,\n 3.0721e-16, 6.7182e-18, 1.7920e-17, 1.3406e-16, 2.3769e-16, 1.4692e-15,\n 6.6721e-16, 2.8110e-16, 2.3135e-17, 1.1048e-15, 2.5921e-15, 2.0943e-15,\n 1.5422e-16, 2.6874e-16, 2.6568e-17, 1.3767e-15, 1.3627e-14, 3.2948e-17,\n 1.2636e-15, 5.8135e-17, 1.2952e-15, 2.0249e-17, 8.4713e-17, 7.0934e-17,\n 2.0474e-15, 9.2663e-16, 5.4746e-17, 2.9089e-15, 6.2376e-17, 3.6744e-17,\n 1.7800e-15, 4.2187e-16, 7.2916e-15, 3.8816e-17, 2.0230e-17, 9.2477e-18,\n 1.7246e-16, 8.1018e-16, 3.5904e-17, 9.3483e-16, 1.5690e-17, 5.2094e-17,\n 5.7362e-15, 5.0929e-15, 2.9436e-17, 2.0967e-14, 1.0153e-15, 2.2305e-15,\n 1.0008e-15, 9.6006e-16, 2.6498e-17, 1.6564e-17, 5.1991e-16, 9.8538e-18,\n 2.2798e-17, 2.4341e-16, 3.2987e-18, 2.3889e-15, 1.9613e-17, 1.6742e-17,\n 1.5080e-15, 2.2127e-17, 2.5911e-16, 3.4141e-18, 7.8618e-18, 5.7286e-17,\n 1.0252e-16, 4.0159e-18, 1.1624e-14, 1.5691e-17, 1.7553e-16, 1.5035e-16,\n 2.2118e-18, 6.1205e-17, 8.7075e-16, 8.3769e-15, 1.3877e-16, 1.5212e-14,\n 4.4458e-15, 2.6853e-16, 1.4646e-17, 1.6065e-15, 5.9850e-18, 1.0156e-14,\n 1.0527e-17, 8.1468e-17, 2.1060e-15, 1.3559e-15, 2.2836e-17, 6.9820e-17,\n 8.1677e-15, 6.1132e-17, 7.6106e-17, 7.3569e-16, 2.9678e-16, 7.4302e-16,\n 1.7654e-16, 2.2115e-17, 7.6478e-16, 3.3453e-17, 4.4334e-16, 8.9670e-16,\n 9.1262e-15, 3.1829e-15, 1.9262e-17, 7.3846e-16, 6.7450e-16, 6.7665e-16,\n 8.4148e-18, 4.1705e-16, 4.3109e-16, 8.2254e-15, 3.7829e-17, 3.1672e-17,\n 2.6271e-16, 2.2113e-17, 1.6528e-15, 2.3453e-17, 2.0739e-17, 5.1674e-16,\n 1.1062e-17, 8.8745e-16, 2.3045e-14, 9.4842e-16, 1.3709e-16, 2.7677e-17,\n 9.3297e-15, 1.5211e-15, 3.0393e-15, 1.1025e-16, 7.3203e-15, 1.8082e-17,\n 3.0395e-14, 7.1226e-15, 4.4719e-17, 3.5238e-16, 1.2112e-15, 1.1621e-15,\n 3.6713e-17, 5.3705e-15, 9.0161e-16, 2.2812e-15, 1.3632e-15, 6.8327e-15,\n 1.8911e-16, 4.1223e-15, 6.5554e-16, 2.0873e-16, 5.9237e-17, 1.0550e-16,\n 4.7772e-16, 5.0938e-18, 1.7280e-15, 4.0003e-17, 1.4005e-16, 6.3106e-17,\n 1.3721e-15, 4.9707e-17, 7.1661e-16, 8.8695e-15, 4.5961e-17, 1.9440e-17,\n 1.5529e-17, 3.2277e-15, 6.3694e-17, 6.4517e-16, 7.1238e-17, 2.4475e-15,\n 2.7327e-16, 6.8990e-15, 2.0241e-17, 9.1248e-18, 2.5939e-16, 3.2222e-15,\n 1.7443e-15, 6.1916e-17, 2.2949e-15, 1.9417e-17, 2.0530e-17, 6.6323e-15,\n 1.2908e-16, 7.6895e-15, 2.3334e-16, 1.7047e-16, 5.5819e-16, 6.1732e-18,\n 2.9861e-17, 2.3418e-17, 3.5323e-17, 2.4218e-17, 1.7173e-16, 4.3563e-18,\n 4.3778e-17, 1.0706e-17, 3.8938e-17, 9.2325e-16, 1.8427e-15, 4.7316e-17,\n 1.5700e-17, 5.3030e-17, 2.0880e-15, 1.2401e-15, 9.8188e-17, 7.1162e-18,\n 4.0855e-17, 2.2721e-17, 1.0164e-16, 1.2864e-16, 3.4596e-15, 1.1094e-15,\n 9.6151e-16, 4.2119e-15, 2.2907e-17, 5.8099e-17, 7.3751e-17, 2.2786e-16,\n 8.1221e-18, 4.6658e-16, 3.7627e-17, 5.1316e-15, 2.1683e-17, 1.1800e-16,\n 1.0106e-16, 4.7888e-15, 3.1166e-18, 1.3624e-15, 1.6448e-15, 1.6025e-16,\n 3.0934e-16, 4.7611e-16, 1.2461e-17, 6.8186e-15, 1.0361e-16, 2.2222e-16,\n 7.3245e-17, 3.2328e-15, 9.7095e-17, 9.8559e-18, 1.6390e-17, 2.3329e-16,\n 6.7247e-16, 3.2807e-17, 3.7230e-14, 7.7870e-15, 4.8882e-18, 3.7382e-15,\n 1.7526e-16, 1.0659e-17, 1.5215e-16, 4.6747e-16, 4.6387e-18, 5.1240e-17,\n 2.4027e-18, 5.9182e-18, 5.4412e-16, 1.2599e-15, 4.0488e-15, 4.4833e-15,\n 4.7538e-15, 8.2959e-15, 8.0754e-15, 3.6150e-17, 2.0916e-16, 9.2797e-17,\n 3.6510e-17, 4.6824e-16, 9.3094e-16, 1.3556e-17, 3.4552e-15, 1.2250e-16,\n 3.7331e-18, 1.0201e-15, 1.1162e-17, 1.6506e-15, 5.8861e-16, 2.0500e-15,\n 2.4775e-15, 5.6831e-16, 1.5357e-17, 8.1432e-17, 2.6890e-18, 1.7978e-15,\n 9.0190e-17, 1.0156e-16, 1.8188e-17, 3.4761e-16, 5.7543e-17, 2.3537e-17,\n 1.1097e-16, 4.5133e-16, 1.9195e-15, 2.5637e-16, 1.6787e-14, 9.7863e-18,\n 1.6983e-16, 2.2825e-16, 5.4212e-15, 3.3668e-15, 3.0324e-16, 3.3660e-16,\n 1.6872e-17, 5.2889e-16, 4.8150e-16, 3.0522e-18, 4.2795e-18, 1.1575e-17,\n 6.2619e-15, 2.5594e-16, 3.6023e-16, 2.2049e-16, 9.7462e-17, 9.7038e-17,\n 1.2281e-17, 9.0345e-18, 1.1339e-15, 3.1005e-15, 8.5551e-17, 2.1709e-15,\n 3.5676e-16, 3.8300e-17, 4.1855e-17, 8.1457e-16, 9.4038e-17, 3.4520e-16,\n 9.9942e-16, 8.7362e-16, 3.2990e-15, 3.2080e-16, 9.2302e-18, 2.4977e-15,\n 2.6813e-15, 5.5732e-16, 3.7739e-17, 1.1793e-16, 1.1252e-15, 2.6106e-17,\n 1.4316e-15, 1.5790e-16, 2.4201e-16, 2.5137e-15, 6.0749e-17, 3.0554e-15,\n 4.2814e-18, 1.7993e-16, 4.9066e-15, 1.2983e-15, 7.3607e-17, 1.1532e-14,\n 4.7826e-16, 5.4015e-17, 3.3149e-17, 1.8303e-18, 1.4923e-15, 7.0976e-17,\n 6.4938e-17, 1.3162e-16, 1.3213e-15, 8.3465e-15, 9.4868e-17, 7.3170e-16,\n 2.5882e-16, 1.2692e-15, 1.4559e-14, 6.4224e-16, 2.2410e-17, 2.1291e-15,\n 2.5811e-17, 2.9746e-15, 1.1714e-17, 2.9925e-15, 1.2796e-17, 7.0123e-15,\n 3.1386e-16, 1.5737e-15, 1.4426e-15, 8.4939e-16, 1.1783e-16, 2.8399e-15,\n 7.0294e-17, 1.4418e-15, 1.4695e-15, 4.6172e-16, 6.3348e-16, 1.4177e-15,\n 1.5252e-17, 1.3263e-17, 6.0550e-15, 4.0887e-14, 1.3871e-16, 5.3306e-15,\n 4.5862e-17, 8.7714e-15, 5.4562e-16, 2.9103e-15, 4.5386e-16, 5.8344e-17,\n 2.9759e-17, 2.1410e-17, 6.1910e-15, 1.1751e-16, 1.3073e-14, 6.2473e-16,\n 6.9508e-15, 6.8905e-19, 2.8859e-16, 1.5517e-14, 1.2769e-17, 3.2648e-16,\n 4.4770e-17, 6.1387e-18, 7.3158e-16, 1.0443e-16, 2.7072e-15, 3.1892e-15,\n 1.6126e-17, 1.3289e-17, 2.2909e-16, 1.3305e-16, 1.7774e-17, 3.2205e-15,\n 1.0353e-15, 3.2549e-15, 3.1374e-17, 1.1949e-16, 7.8707e-17, 5.0031e-17,\n 3.6488e-15, 7.1297e-15, 5.7547e-16, 8.1595e-17, 2.8483e-15, 1.6337e-17,\n 2.5231e-15, 8.5586e-17, 6.2933e-17, 3.1410e-15, 7.7557e-17, 4.2854e-16,\n 1.6528e-16, 4.9930e-17, 2.2242e-15, 7.7148e-17, 3.3326e-16, 4.8035e-17,\n 1.0905e-14, 1.1770e-15, 1.0919e-17, 1.1442e-14, 1.0121e-14, 2.4879e-14,\n 8.9860e-16, 6.4253e-16, 1.4185e-17, 4.2424e-16, 2.4362e-15, 2.0595e-16,\n 7.7118e-16, 7.3709e-15, 5.7500e-17, 2.0153e-17, 5.3686e-15, 1.3251e-16,\n 2.0977e-16, 3.0940e-15, 1.9106e-17, 2.0800e-16, 1.1431e-16, 7.2045e-16,\n 7.2140e-15, 5.8434e-18, 4.3057e-17, 1.0655e-15, 7.2946e-16, 5.5557e-16,\n 4.7884e-15, 1.0020e-16, 1.6590e-16, 2.7249e-15, 4.1774e-17, 7.0822e-16,\n 4.4404e-16, 2.6670e-15, 1.7494e-15, 4.7846e-17, 2.5939e-18, 2.2639e-16,\n 4.2743e-15, 4.0488e-17, 1.0402e-17, 1.2080e-17, 7.1241e-16, 3.8175e-14,\n 5.7543e-16, 3.2874e-15, 9.0138e-15, 8.9290e-16, 4.8635e-17, 5.0156e-16,\n 1.4985e-17, 3.9021e-17, 2.5515e-15, 2.3703e-16, 1.4389e-16, 1.7089e-15,\n 2.9563e-15, 4.4828e-17, 1.8398e-15, 8.7672e-16, 5.8170e-17, 3.1452e-17,\n 7.4245e-15, 5.8965e-16, 4.4025e-17, 3.2358e-17, 6.4267e-16, 1.5412e-16,\n 5.0504e-15, 4.2309e-15, 3.9243e-15, 1.0628e-16, 3.2949e-16, 1.2301e-15,\n 2.6022e-17, 3.9051e-17, 2.7850e-17, 1.3177e-16, 2.3976e-16, 4.2620e-16,\n 4.6088e-16, 3.7189e-15, 4.2788e-15, 9.5556e-17, 2.7013e-15, 3.8070e-17,\n 1.3775e-16, 2.2456e-17, 2.0589e-15, 3.7366e-17, 1.6908e-16, 4.4241e-15,\n 6.8320e-16, 3.7960e-17, 7.3552e-16, 2.2525e-15, 9.9577e-16, 2.0881e-16,\n 2.0445e-14, 3.7123e-16, 2.4914e-15, 7.2688e-16, 7.3972e-15, 2.6173e-17,\n 6.1584e-15, 3.0493e-15, 2.5271e-17, 1.0462e-16, 1.0402e-16, 3.6126e-15,\n 3.2482e-16, 1.0318e-15, 1.4186e-17, 2.4694e-17, 2.4421e-16, 1.6761e-18,\n 5.3921e-15, 6.1003e-16, 2.4457e-17, 9.5823e-16, 2.1073e-15, 8.7064e-17,\n 5.0364e-17, 6.4343e-15, 1.2181e-15, 6.6240e-17, 9.3698e-15, 5.8642e-16,\n 8.3872e-17, 1.2720e-16, 7.9009e-15, 1.2732e-16, 4.3988e-17, 6.2983e-16,\n 9.6077e-17, 3.6501e-17, 2.1324e-16, 7.8955e-18, 3.7465e-17, 4.6276e-15,\n 1.0466e-16, 3.0062e-16, 9.8874e-17, 3.5370e-16, 2.0505e-17, 8.0649e-17,\n 4.4336e-16, 2.4542e-16, 1.7076e-16, 1.4061e-17], device='cuda:0')" + }, + "37": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 4.9771e-13, 1.3566e-13, 2.3587e-13, 1.6913e-13, -5.3482e-15,\n 6.4282e-14, -9.0838e-14, 1.2547e-13, -2.4708e-14, 1.1341e-13,\n -1.9721e-14, 2.8422e-14, 1.5624e-13, -7.8361e-14, 6.5662e-14,\n 1.0693e-13, 3.4790e-13, -7.4591e-15, 8.8857e-14, -2.1235e-14,\n -4.2274e-15, 3.1112e-13, 1.5607e-15, 1.1210e-13, 3.6746e-13,\n -2.1053e-14, 5.2417e-14, 1.3345e-15, 1.5569e-13, 2.2044e-13,\n 5.2331e-14, 2.3938e-14, -5.3551e-15, 3.3219e-13, 1.5600e-13,\n 3.6635e-14, 2.2143e-13, 2.4814e-13, 4.8583e-14, 2.1865e-13,\n -2.8789e-14, 1.0148e-13, 1.0289e-13, -4.5553e-14, -1.4159e-14,\n 4.4572e-14, 1.7103e-13, 2.1354e-13, 1.4862e-13, 3.5289e-15,\n -1.5216e-14, 1.7635e-14, 3.2771e-13, 1.2223e-13, 1.7981e-13,\n 5.4424e-14, 2.2937e-13, 5.1422e-13, 1.3823e-13, -3.8395e-14,\n -1.1782e-14, 1.7275e-13, 1.2275e-13, 2.6735e-14, -1.2347e-14,\n 1.6403e-14, 1.0959e-13, 2.7205e-13, 4.2286e-15, 3.3997e-13,\n 8.2554e-15, -5.0668e-14, 1.1690e-13, 6.9591e-14, 3.2513e-13,\n 2.0383e-13, -2.0769e-14, -6.3875e-15, -4.7241e-15, -1.2418e-14,\n 3.1726e-13, 3.8293e-13, 1.1955e-13, 4.3157e-14, 4.5669e-13,\n 6.6284e-14, 3.2660e-14, 1.0729e-13, 2.4007e-14, -1.8172e-16,\n 1.9335e-13, 2.8212e-13, -1.0437e-14, -3.3350e-15, 2.0193e-14,\n -1.7602e-14, 8.8331e-14, -5.6000e-14, 3.8925e-15, 9.9153e-14,\n 5.1287e-14, 6.7452e-14, 2.2745e-13, 2.6830e-15, 3.0625e-14,\n 3.3493e-15, 1.7617e-13, 1.3131e-13, 8.4582e-14, -1.3366e-14,\n 2.8038e-13, 1.1989e-13, 2.0751e-13, 1.4321e-14, 2.1460e-13,\n -7.9614e-14, 1.4909e-13, 3.7999e-13, -7.4672e-15, 1.2573e-13,\n 1.2714e-13, -7.3156e-14, -2.9379e-14, -3.1124e-15, 3.4055e-13,\n -1.1527e-13, 3.0549e-14, -1.0739e-14, 1.2571e-13, -1.9547e-14,\n -1.5533e-14, -1.5810e-14, 3.6156e-13, 4.8415e-15, -3.5338e-14,\n 1.1111e-13, -7.1285e-15, 4.2110e-14, 1.5001e-13, -1.7362e-14,\n 1.0036e-14, 2.1645e-14, -1.0595e-13, 4.6690e-14, 2.8699e-13,\n 1.9952e-13, 3.4000e-13, 7.8559e-15, 3.6474e-13, 2.9183e-14,\n 4.4007e-14, -2.8946e-15, 4.4938e-13, 4.2079e-13, 1.9296e-13,\n -3.2906e-14, 2.4468e-13, 1.7150e-13, 1.0980e-14, 3.5375e-14,\n 1.0686e-13, 9.3149e-14, 7.3238e-14, 1.9108e-13, 2.1694e-13,\n -5.8074e-14, -2.1834e-14, -7.1939e-15, 3.0941e-13, 8.8814e-14,\n 2.8131e-13, 1.4262e-13, -5.6783e-14, -6.8412e-15, 4.4683e-13,\n 9.8825e-14, 1.8610e-13, -9.9034e-15, -4.4340e-15, 8.8935e-14,\n -3.4851e-15, 3.4827e-13, 1.3779e-13, 3.6427e-13, 1.0139e-14,\n 3.1701e-14, -1.1387e-14, 1.2653e-13, 8.8956e-15, 6.6952e-14,\n 4.5550e-16, 3.9202e-14, 2.0010e-14, -2.5574e-15, -9.2355e-15,\n 6.6237e-17, 3.3064e-14, -2.1895e-14, 7.1996e-14, 1.0639e-13,\n 1.1856e-13, 2.2910e-13, 2.3275e-14, 4.9910e-15, 1.0496e-14,\n -1.9248e-13, -3.1231e-15, 6.0197e-14, -5.3022e-14, 2.4660e-14,\n 1.0998e-13, 3.6713e-13, 1.8732e-13, 7.7890e-14, 2.4836e-14,\n 2.1569e-13, 1.8858e-13, -6.0808e-14, 2.2773e-13, -2.6469e-14,\n -1.2990e-14, 3.5870e-13, -1.4052e-14, -2.4495e-14, -3.9902e-15,\n 1.7884e-13, 1.6569e-14, 5.8160e-15, -6.3247e-16, 9.4670e-14,\n 8.2499e-15, 2.2026e-13, -1.2791e-15, 8.2724e-14, -4.4417e-16,\n 4.3110e-15, -4.8483e-15, -1.2564e-13, 1.0636e-13, 5.1473e-15,\n 1.8327e-15, -3.0640e-14, 1.4831e-13, -1.1232e-13, 6.4903e-15,\n 7.4562e-14, 1.1797e-13, -7.5477e-15, 1.8606e-13, 1.8167e-13,\n 1.3071e-13, 2.6386e-15, -2.8227e-14, 3.6041e-14, -1.7586e-14,\n 8.3284e-14, -8.9152e-15, -9.7449e-15, -4.8380e-15, 1.6948e-14,\n -4.5176e-17, 1.2960e-13, 2.0074e-13, -2.0791e-14, 3.4231e-14,\n -6.8684e-14, 6.0835e-15, 2.1863e-14, 7.3840e-14, 3.6319e-14,\n 2.6476e-13, 3.0544e-13, -4.6300e-14, 5.4840e-14, 7.7057e-14,\n -8.7948e-15, -1.4468e-15, 7.9718e-14, -1.0308e-13, 5.2092e-14,\n 1.3881e-15, -5.2500e-14, -1.2819e-15, 3.2210e-14, 7.7377e-14,\n 4.0842e-13, 7.6679e-15, 1.0605e-13, 2.7669e-14, -2.0639e-14,\n -3.4972e-15, -4.9180e-14, -7.5511e-15, -9.4570e-15, -1.6269e-14,\n 1.1328e-14, 9.7887e-14, -6.6149e-15, 2.5035e-15, 2.0436e-13,\n -1.2619e-13, 1.4845e-13, -6.2656e-14, -2.3324e-14, -2.2972e-15,\n 3.5786e-14, 6.2650e-15, 8.0411e-15, -6.7968e-15, 1.4641e-13,\n 1.0402e-13, -4.1405e-14, -9.6614e-15, 3.2151e-13, 3.7845e-15,\n 5.3125e-14, 6.1163e-14, 4.3578e-13, -6.0477e-14, 4.0293e-13,\n -1.2562e-14, 9.4378e-14, -7.5164e-15, 2.4532e-14, 2.9929e-15,\n -3.9609e-14, 3.1580e-14, 6.3140e-14, -5.4861e-14, 3.9165e-13,\n -6.1730e-15, 1.1137e-13, 1.7564e-13, 1.5040e-15, 7.2163e-13,\n 4.3902e-15, -2.4658e-14, 1.4539e-13, 6.9688e-13, 1.2903e-13,\n -6.7688e-15, 2.1970e-13, 9.1640e-14, 6.5594e-14, 4.3384e-14,\n -1.3545e-14, 6.5719e-15, 4.4439e-13, 3.9888e-14, 4.4157e-13,\n 1.7204e-13, 1.7603e-13, -7.7860e-15, 5.0350e-14, -7.0433e-15,\n -2.6850e-14, 4.4919e-14, 4.4906e-14, -9.7195e-16, 1.2373e-13,\n 1.0672e-14, -2.1849e-14, 1.3751e-14, -5.3229e-14, 1.4647e-13,\n 3.0997e-13, 1.0901e-13, 1.9666e-14, 3.0311e-14, 1.0444e-13,\n -5.7729e-15, 4.0392e-13, 1.7997e-13, -1.7440e-15, 1.3918e-15,\n 3.3082e-14, 1.9413e-13, -4.1905e-14, 5.3089e-14, -1.0379e-14,\n -4.6481e-15, 3.5538e-13, -4.1055e-14, 5.3741e-14, 1.5559e-13,\n -2.6080e-15, 8.4827e-14, 1.4583e-15, -2.5406e-15, 3.9603e-13,\n 3.3233e-13, 9.6403e-14, 2.0188e-15, 2.5301e-13, 3.1334e-14,\n 7.4894e-14, -1.1109e-14, -2.2582e-14, 9.5886e-14, 4.3116e-13,\n 1.3076e-13, 1.1741e-13, 8.9043e-15, 5.2764e-14, 2.1627e-13,\n 8.4528e-14, 2.6783e-14, 1.1254e-13, 1.3038e-13, 2.5570e-13,\n 7.8613e-14, 4.8241e-13, -9.3020e-15, 4.1802e-15, -2.2141e-14,\n 5.0579e-14, 4.1781e-13, 2.2807e-13, 2.2835e-13, 4.4428e-14,\n 2.0328e-14, -1.5853e-15, 1.2597e-13, 7.6141e-14, 1.5476e-13,\n 1.0684e-13, 1.9262e-13, -3.3152e-15, 3.7452e-13, -1.7237e-13,\n 6.2471e-14, 1.4222e-13, -1.9214e-14, 3.6347e-13, 1.7456e-13,\n 3.0172e-13, 1.1512e-13, 5.1111e-14, -3.2270e-14, -3.8844e-14,\n 3.5437e-14, 1.8396e-13, 2.3335e-13, 1.6058e-13, 3.8884e-13,\n -4.9798e-15, -1.3759e-13, 3.2005e-14, 5.6506e-14, 3.7764e-14,\n -2.3621e-16, 3.5625e-14, 3.0710e-14, 1.3632e-13, 7.7782e-14,\n 2.6563e-13, 2.8354e-15, -1.0243e-14, 2.6996e-14, -4.2816e-15,\n 1.6922e-13, 2.2776e-13, 2.0334e-14, 1.2762e-13, -3.6095e-15,\n -3.3823e-14, -1.4773e-15, 1.1141e-13, 3.1006e-14, -2.4833e-14,\n 9.1349e-14, 2.6286e-13, 3.0556e-14, 1.2993e-13, 1.6717e-13,\n 1.0365e-13, 9.5984e-16, 3.5259e-13, -5.1911e-15, 4.2229e-14,\n 2.5929e-14, -4.6486e-16, 3.8136e-13, 6.5206e-17, 3.8154e-13,\n -1.1109e-14, 6.4773e-13, 1.3809e-13, -1.1695e-14, 9.1051e-14,\n -2.7887e-14, 5.1630e-13, 2.4849e-14, -3.4926e-14, 2.6971e-15,\n 2.4536e-13, 1.9178e-14, 6.7350e-14, -1.2342e-13, 1.8189e-13,\n 3.2681e-15, -2.4192e-14, 2.5456e-13, -6.8445e-15, -2.8869e-15,\n 7.4608e-14, 1.7157e-14, 8.3631e-14, 2.1517e-13, 1.7516e-14,\n 7.4777e-14, 1.3709e-13, -9.4863e-15, -2.2223e-13, 1.9477e-13,\n 1.0788e-14, 3.7010e-13, 3.6662e-15, 6.6177e-14, 2.2977e-13,\n 1.1178e-13, 2.5300e-13, -3.7860e-14, 2.7118e-14, 2.3746e-13,\n -1.7469e-15, -7.2284e-15, 2.6548e-13, -5.9393e-15, -9.5620e-15,\n 3.4465e-14, 6.1111e-14, -1.0179e-14, 9.0300e-13, 1.7429e-13,\n 3.8054e-13, 3.0043e-13, 2.2707e-13, -5.0016e-15, 8.0835e-14,\n -6.4253e-15, 2.2329e-14, 1.4545e-13, 4.3648e-13, 7.7118e-14,\n 9.5844e-14, 4.7090e-13, -1.3984e-14, 1.3761e-13, 1.6149e-14,\n -1.1197e-14, -5.2849e-14, 2.3527e-13, 6.3817e-15, -4.9240e-15,\n -1.6090e-14, 8.7227e-14, 3.1393e-13, 2.2072e-13, 1.3959e-13,\n 6.4424e-13, 2.7233e-14, 2.3269e-13, 1.7453e-13, -3.9820e-15,\n -5.4788e-15, 8.4674e-14, 2.6509e-15, 2.4373e-13, 7.8695e-14,\n 2.8272e-13, 4.6265e-13, 4.8763e-13, 1.2444e-14, 1.5295e-13,\n 1.2905e-15, -6.0011e-16, -6.3458e-15, 1.7643e-13, -3.3874e-14,\n 1.8511e-13, 1.1262e-13, 3.8211e-13, -1.3163e-14, 2.2039e-13,\n 1.8434e-13, 8.3963e-14, -2.2238e-14, -1.3779e-13, 2.7898e-13,\n 5.3864e-14, -9.7373e-16, 4.2130e-13, 3.8483e-14, 3.0908e-14,\n 2.6851e-13, 3.2586e-14, 1.0501e-14, -1.0170e-14, -9.4800e-15,\n 1.4358e-13, 2.2217e-13, -1.3435e-14, 2.2266e-13, -2.9956e-15,\n -3.1513e-14, -1.2655e-13, 4.3065e-13, 1.5606e-13, 1.3785e-13,\n 1.2257e-13, 3.7747e-14, 1.0709e-13, 3.6139e-13, 1.0213e-14,\n 2.3454e-15, 3.1533e-14, 1.6882e-13, 2.7489e-14, 3.7850e-14,\n 1.2887e-13, 2.5139e-13, -3.7031e-14, -3.0838e-14, 3.3773e-14,\n 1.1482e-14, 2.4698e-13, 2.7538e-14, 4.3593e-13, 1.1868e-13,\n -2.7133e-14, 1.6001e-13, -9.5523e-15, 3.3053e-13, 3.3799e-14,\n 3.1934e-13, 4.6957e-13, -4.1938e-14, 2.2211e-14, 2.0422e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.4897e-15, 9.5685e-16, 2.3522e-15, 4.0547e-15, 4.3964e-18, 2.0362e-15,\n 1.7016e-16, 3.0445e-16, 3.4908e-16, 2.3440e-15, 5.9537e-15, 1.9128e-16,\n 2.1035e-15, 2.0899e-17, 1.0273e-15, 1.2414e-15, 1.1574e-15, 1.1582e-15,\n 1.0478e-15, 1.5531e-16, 6.6249e-18, 1.1506e-15, 2.2644e-15, 2.4384e-15,\n 2.2063e-15, 2.4716e-17, 9.7297e-16, 1.6019e-15, 4.1821e-15, 4.3214e-16,\n 2.6590e-15, 8.0169e-16, 4.5681e-18, 1.2220e-15, 1.4774e-15, 2.0955e-15,\n 5.8129e-17, 5.5597e-15, 1.6115e-15, 4.4270e-15, 7.4253e-17, 1.2586e-15,\n 8.9419e-16, 3.1615e-16, 1.1318e-17, 6.4636e-16, 3.1671e-15, 3.2873e-15,\n 1.3062e-15, 1.2589e-15, 1.7656e-17, 2.0959e-15, 1.4569e-15, 1.4189e-15,\n 5.5579e-16, 1.4690e-15, 2.2995e-17, 1.1778e-15, 4.2043e-15, 2.1995e-17,\n 1.8137e-15, 3.0814e-17, 2.6689e-15, 2.7039e-16, 3.0986e-17, 4.4758e-16,\n 1.9523e-15, 3.7203e-15, 2.7591e-17, 2.1257e-15, 2.4373e-17, 1.8449e-17,\n 4.6495e-15, 1.7208e-15, 5.9880e-15, 7.0475e-16, 6.2170e-18, 3.2034e-17,\n 4.7399e-17, 2.3595e-15, 6.0114e-16, 9.1567e-16, 3.8664e-16, 4.8372e-16,\n 4.7400e-15, 2.2544e-15, 1.5436e-16, 8.2005e-15, 1.5607e-15, 2.1778e-15,\n 1.1687e-15, 1.7103e-15, 6.6503e-18, 3.1227e-16, 1.2792e-15, 6.3973e-18,\n 1.3246e-16, 9.3616e-16, 7.7040e-17, 1.2773e-15, 1.1822e-16, 1.3482e-16,\n 3.7493e-15, 9.2886e-18, 1.5805e-15, 2.6861e-16, 2.7856e-16, 3.7288e-16,\n 9.6837e-16, 1.2395e-16, 4.5688e-15, 5.5379e-17, 3.2447e-16, 8.4780e-16,\n 4.5954e-17, 2.9822e-16, 1.1017e-15, 4.1992e-15, 6.2405e-17, 4.7653e-15,\n 4.0443e-15, 6.9877e-16, 3.2684e-17, 2.5771e-15, 6.2149e-17, 4.7801e-15,\n 6.3041e-16, 5.3639e-16, 2.2841e-15, 3.3288e-15, 6.2371e-17, 7.2783e-16,\n 3.8487e-15, 3.5186e-17, 1.0325e-15, 1.7297e-15, 1.5061e-15, 1.2195e-15,\n 8.8891e-16, 3.7231e-16, 1.2252e-15, 7.0023e-16, 1.4059e-15, 1.8321e-15,\n 8.1198e-15, 3.9519e-15, 7.5097e-17, 2.0095e-15, 7.7286e-16, 7.2581e-16,\n 3.0199e-16, 1.4928e-15, 1.2519e-15, 3.2636e-15, 5.2111e-16, 1.7392e-17,\n 4.4128e-16, 3.7308e-16, 2.3891e-15, 1.0118e-16, 5.4770e-16, 9.7997e-16,\n 1.0685e-17, 2.2425e-15, 8.3346e-15, 2.5502e-15, 5.0500e-16, 3.4629e-16,\n 5.6884e-15, 2.4531e-15, 3.9052e-15, 7.7390e-16, 4.7118e-15, 3.2455e-18,\n 1.8590e-14, 4.7250e-15, 8.3069e-16, 1.0272e-15, 2.8926e-15, 1.6745e-15,\n 9.5538e-18, 5.2161e-15, 2.9737e-15, 3.6822e-15, 3.1917e-15, 7.6614e-15,\n 7.5022e-16, 1.6844e-15, 9.6956e-16, 1.1669e-15, 2.9718e-17, 2.4233e-15,\n 2.4518e-15, 1.7539e-16, 1.8278e-15, 1.0379e-17, 5.0112e-16, 4.5160e-16,\n 4.0032e-15, 8.5241e-17, 1.6926e-15, 3.3494e-15, 5.3041e-16, 8.5780e-18,\n 2.0253e-16, 3.7799e-15, 1.9537e-17, 1.6517e-15, 3.6148e-17, 1.7507e-15,\n 6.3269e-16, 2.5712e-15, 2.9210e-16, 1.6622e-16, 6.0158e-16, 3.6582e-15,\n 1.4992e-15, 1.6340e-17, 1.6591e-15, 2.3445e-16, 3.4267e-16, 2.3646e-15,\n 5.5174e-17, 6.4470e-15, 7.1019e-16, 1.3560e-15, 1.5754e-15, 1.7613e-16,\n 5.5742e-18, 5.9549e-16, 2.1990e-17, 3.6333e-16, 1.1419e-15, 2.4512e-16,\n 1.1183e-17, 4.2530e-17, 3.8221e-16, 1.4110e-15, 2.1343e-15, 2.9299e-16,\n 1.9945e-16, 3.2146e-16, 5.2359e-15, 1.1208e-15, 3.9023e-16, 2.7503e-16,\n 8.6534e-17, 1.9368e-16, 1.0734e-15, 7.8408e-17, 1.3361e-15, 1.2902e-15,\n 1.1638e-15, 5.2798e-15, 1.4811e-17, 3.1740e-17, 8.0870e-16, 9.7264e-17,\n 1.7506e-16, 1.0957e-15, 1.0696e-17, 3.9661e-15, 3.0687e-16, 6.4982e-17,\n 9.0259e-16, 3.0185e-15, 1.6498e-16, 2.6762e-15, 2.5192e-15, 5.0558e-16,\n 4.1248e-16, 1.0678e-15, 5.3407e-16, 3.6999e-15, 8.8124e-16, 7.0946e-16,\n 2.6318e-17, 3.8157e-15, 7.2149e-16, 7.8694e-16, 3.3442e-18, 1.0696e-15,\n 2.3868e-15, 3.5219e-16, 1.6025e-14, 6.7981e-15, 9.4218e-17, 3.8814e-15,\n 7.9254e-17, 7.5074e-17, 1.1905e-15, 1.5893e-15, 3.5563e-17, 1.0592e-15,\n 6.6079e-18, 1.3329e-16, 2.2630e-15, 1.8321e-15, 4.6824e-15, 3.3168e-15,\n 3.0492e-15, 2.1140e-15, 5.4404e-15, 8.0563e-16, 1.1884e-15, 5.7472e-16,\n 6.5927e-16, 3.1697e-15, 1.8769e-15, 8.1177e-17, 4.1740e-15, 4.1332e-16,\n 1.4488e-16, 4.9555e-15, 5.8275e-16, 3.8378e-15, 2.0870e-15, 1.7301e-15,\n 1.7956e-15, 1.9998e-15, 2.4088e-16, 1.7686e-16, 5.8104e-18, 2.8251e-15,\n 4.3724e-16, 3.5303e-16, 3.8832e-16, 2.2905e-15, 4.6408e-17, 1.8516e-16,\n 7.5779e-17, 1.7241e-15, 2.6937e-15, 8.2328e-16, 1.1248e-14, 1.5215e-16,\n 8.1035e-17, 9.5641e-16, 5.0782e-15, 2.3381e-15, 7.7808e-16, 8.5207e-16,\n 3.0919e-16, 1.2515e-15, 8.0679e-16, 2.6834e-16, 4.8347e-17, 6.5852e-16,\n 2.2542e-15, 1.4206e-15, 1.1011e-15, 1.4095e-15, 2.6289e-17, 4.6019e-17,\n 3.4752e-16, 1.8652e-16, 1.0702e-15, 2.5971e-15, 2.5030e-17, 1.0339e-15,\n 9.1936e-16, 2.5513e-16, 1.9463e-16, 8.0303e-16, 3.1526e-16, 6.9816e-16,\n 1.7056e-15, 2.0363e-15, 3.5778e-15, 1.0169e-15, 1.7882e-16, 2.9138e-15,\n 1.5990e-15, 8.9289e-16, 1.7947e-16, 4.3664e-16, 1.7661e-15, 5.6169e-16,\n 4.9763e-15, 1.8635e-15, 7.7699e-16, 2.2649e-15, 1.8667e-17, 3.4441e-15,\n 2.8317e-16, 5.5103e-17, 2.8776e-15, 1.5489e-15, 5.1899e-17, 3.0879e-15,\n 8.5176e-16, 7.8910e-16, 5.0613e-17, 7.1408e-17, 2.5642e-15, 2.2790e-17,\n 2.2514e-17, 4.1498e-17, 1.1875e-15, 1.9642e-15, 8.7272e-16, 1.5898e-15,\n 7.9817e-16, 3.6343e-15, 4.8830e-15, 1.1907e-15, 3.4929e-16, 1.9074e-15,\n 1.9569e-16, 2.9264e-15, 3.9533e-16, 3.1762e-15, 4.6349e-18, 6.3594e-15,\n 8.0438e-16, 1.5692e-15, 2.0831e-15, 2.7497e-15, 1.5951e-15, 1.7495e-15,\n 5.5016e-16, 2.3787e-15, 2.0707e-15, 9.3523e-16, 1.6483e-15, 1.9775e-15,\n 3.4249e-16, 4.1029e-16, 4.0805e-15, 1.3715e-14, 3.9391e-16, 4.5051e-15,\n 1.1664e-17, 9.1634e-15, 9.3981e-16, 3.3001e-15, 1.9834e-15, 2.2586e-17,\n 8.1716e-18, 6.8982e-17, 1.8414e-15, 4.4809e-16, 5.0719e-15, 1.7956e-15,\n 7.6768e-15, 8.5679e-17, 1.8559e-15, 8.0919e-15, 4.6985e-16, 1.0263e-15,\n 1.9550e-17, 1.0216e-16, 2.2512e-15, 8.1692e-16, 3.2769e-15, 2.0263e-15,\n 7.4724e-17, 4.6971e-16, 5.4574e-16, 5.6585e-17, 1.0388e-15, 4.4178e-15,\n 2.1486e-15, 2.3320e-15, 6.1711e-18, 1.6186e-15, 4.1868e-16, 1.6627e-17,\n 2.1374e-15, 2.6903e-15, 8.8564e-16, 2.3836e-16, 3.6345e-15, 1.7364e-17,\n 3.3757e-15, 3.9542e-16, 4.5588e-16, 5.4135e-15, 5.5228e-17, 4.5770e-16,\n 8.3744e-16, 7.5742e-16, 2.1930e-15, 2.2404e-17, 1.9068e-15, 1.4236e-17,\n 7.3135e-15, 1.5460e-15, 2.4368e-16, 4.3353e-15, 8.2017e-15, 1.4995e-14,\n 1.1427e-15, 1.7078e-15, 2.6184e-16, 7.1945e-16, 3.3385e-15, 8.1998e-16,\n 3.0601e-15, 1.9730e-15, 2.4246e-17, 3.9839e-18, 2.6456e-15, 5.0159e-17,\n 9.7832e-16, 3.1507e-15, 3.7648e-16, 9.9773e-16, 5.4751e-16, 8.7515e-16,\n 6.3447e-15, 2.6296e-16, 2.2921e-17, 5.7357e-15, 1.9345e-15, 3.0895e-15,\n 1.9183e-15, 2.9269e-17, 9.3431e-16, 1.8621e-15, 2.7154e-17, 4.8472e-16,\n 1.0527e-15, 2.1092e-15, 2.8100e-15, 1.1368e-17, 9.8399e-17, 1.4565e-15,\n 2.2999e-15, 6.1500e-16, 2.6399e-16, 1.1857e-16, 2.4962e-15, 1.4910e-14,\n 1.9532e-15, 2.6747e-15, 3.6548e-15, 1.1128e-15, 1.8231e-17, 7.7527e-16,\n 6.6472e-16, 4.6360e-16, 1.1750e-15, 6.9359e-16, 7.6400e-16, 1.1982e-15,\n 1.8705e-15, 1.0342e-17, 3.5575e-15, 1.7125e-15, 2.9866e-17, 1.3863e-17,\n 2.4715e-15, 1.1094e-15, 1.9265e-17, 1.2549e-17, 1.9590e-15, 8.6360e-16,\n 4.6462e-15, 4.5146e-15, 7.8099e-15, 9.8621e-16, 8.7949e-16, 2.5525e-15,\n 2.5644e-16, 8.0204e-18, 5.4349e-16, 4.9506e-16, 1.0062e-15, 1.8761e-15,\n 7.2415e-16, 2.3682e-15, 6.5955e-15, 9.2110e-16, 2.1381e-15, 1.4201e-17,\n 4.1785e-16, 1.3116e-16, 2.7836e-15, 1.1590e-17, 4.6207e-16, 2.6752e-15,\n 2.5531e-15, 8.2056e-18, 1.1059e-15, 3.8762e-15, 2.4416e-15, 1.7574e-15,\n 1.0778e-14, 7.7155e-16, 4.3699e-15, 2.1385e-15, 3.4369e-15, 2.7241e-16,\n 4.1566e-15, 3.3316e-15, 2.8814e-16, 4.4602e-16, 2.5085e-17, 2.2280e-15,\n 4.3641e-16, 2.8618e-15, 1.2545e-16, 2.4139e-16, 8.3640e-17, 1.6109e-16,\n 3.2447e-15, 1.3132e-15, 5.0262e-17, 3.0152e-15, 1.6280e-15, 7.8440e-16,\n 1.2174e-15, 3.1057e-15, 2.8020e-15, 3.2768e-17, 4.1707e-15, 7.2916e-16,\n 1.0352e-16, 5.4999e-16, 2.7019e-15, 2.1045e-16, 1.3617e-17, 3.0993e-15,\n 3.2307e-16, 1.0400e-17, 1.5293e-15, 1.3416e-16, 7.6679e-16, 4.2117e-15,\n 4.7257e-17, 5.0296e-16, 4.7827e-16, 5.7486e-16, 5.7478e-16, 1.7990e-15,\n 4.4466e-16, 1.3204e-15, 5.8399e-17, 6.2491e-17], device='cuda:0')" + }, + "38": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, 5.9533e-17, 1.0991e-19, ..., -4.9890e-17,\n -1.2616e-15, 2.7729e-17],\n [ 5.6052e-45, -3.9958e-16, 3.8126e-18, ..., -2.8278e-16,\n -5.4295e-16, 1.3945e-17],\n [ 5.6052e-45, 1.9308e-16, -1.2079e-17, ..., -1.3134e-16,\n 2.8594e-16, 1.3187e-17],\n ...,\n [-5.6052e-45, -1.5180e-16, -1.5762e-18, ..., -2.2092e-15,\n -3.0358e-16, -5.1312e-17],\n [ 5.6052e-45, 3.4896e-16, -2.5259e-17, ..., -1.1613e-15,\n -3.0645e-15, -4.4140e-17],\n [-5.6052e-45, -2.9199e-16, -5.9826e-18, ..., -1.2301e-17,\n -3.0763e-15, -9.5747e-18]], device='cuda:0')", + "exp_avg_sq": "tensor([[4.0491e-16, 6.9868e-16, 1.5264e-15, ..., 1.6244e-18, 3.6035e-17,\n 8.5383e-16],\n [1.8856e-15, 2.9432e-15, 4.1216e-15, ..., 2.4464e-18, 1.4293e-16,\n 3.1086e-15],\n [2.3594e-16, 1.6105e-16, 4.1569e-16, ..., 1.7257e-18, 1.1067e-17,\n 1.4709e-16],\n ...,\n [1.2567e-15, 1.3267e-15, 1.1806e-15, ..., 2.0905e-17, 2.4661e-16,\n 1.3365e-15],\n [6.4410e-16, 1.5737e-15, 1.4097e-15, ..., 2.0150e-17, 1.1956e-16,\n 9.0816e-16],\n [2.6802e-15, 9.9708e-15, 1.6214e-14, ..., 3.2623e-17, 2.8487e-16,\n 7.2166e-15]], device='cuda:0')" + }, + "39": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.2691e-13, -1.1957e-12, -1.3675e-12, 1.3907e-12, -7.0585e-13,\n -1.1046e-12, -3.9893e-12, 2.0693e-12, -6.4762e-13, -9.8616e-13,\n -1.5542e-12, -2.4056e-12, 4.4684e-12, -1.2737e-12, -1.1054e-12,\n -3.4908e-13, 2.2123e-12, -9.2254e-13, 9.7016e-13, -3.5718e-12,\n -2.5002e-13, 3.8632e-13, -1.9654e-12, -9.6564e-13, 8.1629e-12,\n 1.9048e-12, 1.7974e-12, -5.7934e-13, 3.9928e-12, -9.4919e-13,\n -1.0391e-12, -1.0078e-12, -7.8419e-13, -1.0230e-12, -7.5120e-13,\n -1.8894e-12, -1.0569e-12, -9.6474e-13, -9.6642e-13, 1.9591e-12,\n 8.6092e-13, 2.6238e-12, -1.4133e-12, -1.9097e-12, 2.3344e-12,\n -4.2757e-13, 2.4716e-12, -1.4425e-12, 2.1628e-12, -1.1327e-12,\n 2.6879e-12, -2.6883e-12, -1.5320e-12, -1.5907e-12, 1.8027e-12,\n 3.5094e-12, 3.6102e-14, 3.7451e-12, 6.4375e-13, 4.5261e-12,\n -4.3174e-12, 9.8121e-13, 2.5073e-12, 4.0338e-13, -1.1709e-13,\n -1.2683e-13, -2.2147e-12, -9.5250e-13, -1.0075e-12, 6.0688e-14,\n -8.8454e-13, -6.8591e-13, 4.0171e-14, -1.3029e-12, 1.9563e-12,\n -2.3167e-12, -3.5138e-13, 1.0910e-12, 3.2649e-13, -1.4018e-12,\n 1.6860e-13, 2.4979e-12, 2.2334e-12, 5.0929e-12, -1.7109e-12,\n -1.9041e-12, 1.1063e-13, 3.3747e-13, -2.4962e-12, -1.0846e-13,\n 5.2617e-13, -7.5081e-13, -3.5817e-12, -4.4190e-12, -1.3436e-13,\n -1.4555e-12, -1.4694e-12, -3.2272e-12, -1.6256e-12, 1.6375e-12,\n -2.5426e-12, -1.5546e-12, 8.0747e-12, 9.1759e-13, -1.2794e-12,\n -2.9950e-12, -5.3934e-13, 1.6992e-13, 1.4325e-13, -9.0575e-13,\n -4.9965e-13, -1.2853e-12, -1.0388e-12, -1.6191e-12, 3.1345e-12,\n -1.2943e-12, -1.1303e-12, 3.7526e-12, -2.1621e-12, 2.5168e-12,\n -2.7526e-13, -3.6352e-12, 3.8140e-12, -6.9118e-13, 1.8195e-13,\n 4.9972e-13, -1.0365e-12, -2.2283e-12, 8.5206e-14, 3.0409e-13,\n 9.4673e-13, -2.2974e-12, 3.8847e-12, 6.7371e-13, 1.1027e-12,\n -6.7034e-13, 2.2151e-12, 2.2735e-13, -3.8417e-13, -9.0666e-13,\n 6.7641e-13, 6.2337e-12, -4.3153e-12, -7.2151e-13, -9.3837e-13,\n 3.2312e-13, 1.1697e-12, -8.3077e-13, 6.1459e-12, 7.1396e-13,\n 6.5566e-13, -3.6308e-13, 5.5589e-12, 1.4482e-12, 6.8136e-12,\n -4.4314e-14, 4.0994e-12, -4.0911e-13, -1.2161e-12, -2.9857e-13,\n 1.8827e-12, -8.3253e-13, 3.7822e-12, -5.0017e-13, -9.0507e-13,\n 1.9281e-12, 1.9393e-12, -8.8214e-13, 2.9073e-12, 1.7614e-12,\n 9.5117e-12, 7.8690e-13, -3.7605e-12, -2.6721e-12, -9.2267e-13,\n -1.1591e-12, 3.4869e-13, -1.8786e-12, -6.9294e-13, -1.3022e-12,\n 2.5711e-12, -1.1107e-12, 1.2834e-12, 3.8333e-14, -1.8538e-12,\n 8.5583e-13, 1.3441e-12, -1.4465e-12, -7.4973e-13, -3.7718e-12,\n -1.4168e-12, 2.4136e-12, 1.0437e-12, 1.5063e-12, -1.5076e-12,\n -1.9365e-12, -1.0474e-12, -8.3782e-13, -1.0431e-12, -6.5128e-13,\n 2.6979e-12, -1.5574e-12, 2.0490e-13, -2.4778e-12, 1.6927e-12,\n -1.1516e-11, 8.3256e-13, -2.5854e-12, 7.4061e-12, -4.1080e-12,\n 4.4061e-13, 8.8785e-13, -8.8171e-13, -1.0059e-12, -4.4866e-13,\n -3.3597e-13, -6.9549e-13, 7.1292e-12, 9.8567e-13, 6.3469e-13,\n -8.0810e-13, 3.4130e-14, 1.9424e-12, -9.7071e-13, -2.2939e-12,\n 1.7868e-12, -9.1034e-14, 7.7363e-13, 2.2011e-12, 1.9983e-12,\n 1.1728e-12, 3.9013e-12, -1.9524e-12, 5.6029e-12, -9.6397e-13,\n -2.4485e-12, -1.9233e-12, -4.1672e-12, 3.1477e-12, -5.0779e-13,\n -1.2704e-12, -1.7100e-13, -5.4692e-13, 1.5455e-13, -2.0924e-13,\n 8.9655e-14, 3.9561e-12, -2.0678e-12, 1.4186e-12, -3.1114e-13,\n -1.3707e-12, -3.6719e-12, 7.3829e-12, 1.1109e-12, -1.1932e-12,\n -9.0987e-13, 1.9250e-12, -1.1248e-12, -1.2015e-13, -7.7566e-13,\n -3.9415e-12, -1.9042e-13, -9.9972e-13, 6.3075e-13, 8.7206e-13,\n -2.8601e-12, -2.7747e-12, -9.5013e-13, -4.6202e-13, 1.5911e-12,\n 2.7718e-12, 1.8530e-12, -1.0984e-12, 5.1115e-15, 1.5946e-12,\n -4.1400e-13, -2.3268e-12, -3.0798e-13, -7.4339e-14, -8.2414e-13,\n -3.8716e-12, 6.1921e-12, -1.9071e-13, -2.4053e-12, -5.1864e-14,\n -2.6608e-13, -3.0567e-12, -9.9003e-13, -9.8827e-12, -2.5230e-13,\n 2.8270e-12, -6.6291e-13, -2.9361e-12, -1.1359e-12, 7.5142e-13,\n 1.2040e-12, 1.2078e-13, -5.7583e-12, -2.2107e-12, 2.8127e-12,\n -3.2491e-12, -1.0618e-12, -3.9690e-12, -1.1027e-13, 2.9124e-12,\n -3.5035e-13, 3.0502e-12, -2.0424e-13, -1.2281e-12, 4.1337e-13,\n -3.7638e-12, 8.9621e-12, 3.7179e-13, 9.7669e-12, -1.9043e-13,\n -1.0686e-12, -5.9509e-13, -1.4243e-12, -3.9074e-12, 8.5200e-12,\n -1.2168e-12, 3.6238e-12, -1.8698e-12, -1.7059e-12, 1.8280e-12,\n -5.0413e-13, -2.2325e-13, -1.5522e-12, -1.0040e-12, 5.5453e-12,\n -1.0583e-12, 4.1994e-12, -1.2059e-12, -1.6843e-12, 2.5804e-12,\n -6.1605e-13, -8.7138e-13, 4.9291e-13, 2.1337e-12, -3.8058e-15,\n 4.4783e-13, -9.6123e-13, -1.2963e-12, -9.6144e-13, -3.8946e-12,\n -4.6537e-12, 3.6876e-12, -8.6200e-13, -8.4311e-13, -7.2767e-13,\n -1.1179e-12, 2.0338e-12, 1.9533e-12, 2.5616e-12, 1.5019e-12,\n -4.0780e-13, 2.2388e-12, -1.6332e-12, -1.8870e-12, -4.0180e-13,\n -1.4078e-12, 3.0512e-12, -9.7241e-13, -2.4171e-12, 1.8049e-12,\n 1.5610e-12, -7.6756e-13, -8.4606e-13, -2.3729e-12, -1.0359e-12,\n -7.2031e-13, 3.2365e-12, 2.5369e-12, -5.7418e-13, -1.0851e-12,\n 3.7038e-12, -5.7736e-13, 1.3942e-12, -8.2499e-13, -1.9757e-12,\n -1.2335e-12, 3.4041e-12, 2.3283e-12, -1.0982e-12, 3.9566e-12,\n -2.6933e-12, -1.1243e-12, 3.5257e-13, -3.2525e-13, 2.6475e-12,\n -1.1921e-12, 2.6940e-12, -4.5911e-12, -1.0370e-12, -8.1921e-13,\n -1.4890e-13, -2.0831e-14, -6.4665e-13, 1.2271e-13, -8.7149e-14,\n -3.5079e-13, -9.7464e-13, 7.3701e-13, -2.8977e-12, 5.6913e-12,\n -8.0899e-13, -3.9333e-13, -1.2283e-12, -1.3291e-12, 3.1412e-13,\n -1.0021e-12, 7.0955e-12, -5.1135e-13, -3.1555e-12, -2.2790e-12,\n 3.4949e-13, 1.1878e-13, -5.0981e-13, -8.6836e-13, -8.6612e-13,\n -9.7120e-13, -5.2526e-13, -1.8229e-12, -1.1317e-12, -7.7959e-13,\n 2.9971e-13, 3.9936e-12, -3.8541e-13, -1.0305e-12, 2.5906e-12,\n 1.0822e-12, 1.0498e-12, -9.8873e-13, 1.5678e-12, -1.3877e-12,\n 2.4035e-12, -1.6903e-12, 9.2748e-13, 2.0636e-12, -1.3327e-12,\n -3.4378e-13, -1.5326e-12, -1.0579e-13, -1.8431e-12, -1.2748e-13,\n 8.2856e-14, -9.1049e-13, -3.6189e-13, -1.0296e-12, -3.3705e-12,\n -1.7102e-12, 7.9179e-13, 2.1694e-13, 3.0371e-12, -8.9524e-13,\n -1.4329e-12, 1.6212e-12, 1.4984e-12, 5.1022e-12, -8.7919e-13,\n 1.3448e-12, 1.0951e-12, -3.9907e-13, -6.6964e-13, -4.9544e-14,\n -3.9684e-12, 9.5992e-13, 2.6705e-12, -1.0917e-12, 1.6103e-12,\n -3.8851e-12, 4.1900e-12, -1.6438e-12, 6.9093e-13, -9.1626e-13,\n -3.0111e-12, -3.3587e-12, -1.0036e-12, -2.6421e-13, -9.7958e-13,\n 1.0559e-12, -1.5446e-12, 8.4287e-12, -6.2935e-12, -9.7600e-13,\n -2.4737e-13, 2.0618e-12, -7.3697e-13, 3.7959e-12, 3.3087e-12,\n 1.3005e-11, 3.1911e-12, -1.1434e-12, 3.2192e-13, 1.5863e-12,\n -1.3209e-12, 1.6495e-12, -3.6687e-12, -8.6987e-12, -3.5852e-12,\n -1.1987e-12, -9.4159e-13, 1.3957e-12, -8.9211e-14, -3.7730e-12,\n -1.4575e-12, -1.4483e-12, -3.8376e-13, -9.4435e-13, 1.0489e-12,\n -1.9433e-12, -6.7746e-13, -7.4697e-13, -2.9142e-12, -1.6600e-13,\n -1.3739e-12, 1.9806e-12, 2.2004e-13, -4.4293e-13, 8.4408e-13,\n -8.4741e-14, -8.8004e-13, 7.7173e-13, -1.2268e-12, -9.1252e-13,\n -1.0077e-12, -4.9430e-12, 1.5673e-14, -2.2564e-12, 3.2453e-13,\n -1.6987e-12, 1.7605e-12, -3.5397e-12, 6.3256e-12, 3.3688e-12,\n 3.5906e-12, 1.5251e-12, -1.3462e-12, -1.1632e-13, 1.8528e-12,\n -6.2549e-13, -9.8047e-13, -1.0013e-12, -3.3234e-13, -1.0981e-12,\n -1.0314e-12, -8.7368e-14, 7.6561e-14, -2.8501e-13, -1.1654e-13,\n 1.6556e-12, 1.3935e-13, 1.0287e-12, -1.1401e-12, -6.6440e-12,\n -1.5419e-12, -2.6129e-13, -6.7197e-13, 2.4715e-12, -6.1194e-13,\n 1.3330e-11, -2.2880e-12, -6.0011e-13, -1.6006e-13, -2.1642e-12,\n 1.4404e-12, 4.1883e-12, -1.6523e-12, -1.6135e-14, 2.4048e-13,\n -9.0380e-13, 9.8832e-13, 1.4217e-12, -8.9010e-13, -1.0329e-12,\n -1.5019e-12, -1.2369e-12, -6.4772e-13, 6.1801e-13, 1.5243e-12,\n -1.1116e-12, -3.5729e-12, -1.3198e-12, -9.2422e-13, -1.9980e-12,\n 1.9406e-12, -9.0443e-13, 9.0673e-13, 1.8447e-13, 6.8008e-13,\n 1.7671e-13, -9.9883e-13, -8.4806e-13, 2.9843e-12, -6.1062e-13,\n 4.2501e-12, -1.2105e-12, 4.1821e-12, 6.4749e-13, 1.6724e-13,\n -3.8730e-14, 1.7035e-12, -7.6693e-13, -1.2636e-12, -2.0548e-12,\n -8.4424e-13, -4.2490e-12, 1.4474e-12, -2.2909e-12, -4.7687e-13,\n 2.2875e-12, -3.6852e-12, -1.5331e-12, 3.2583e-12, -1.0011e-12,\n -1.3693e-12, -8.6913e-13, -2.0888e-12, -1.0855e-12, 4.2388e-12,\n -2.0895e-12, 3.1050e-13, 1.4757e-13, 3.4845e-12, -4.0882e-13,\n -7.3572e-12, 4.2863e-12, -1.1058e-12, 2.3350e-12, 5.2190e-12,\n 4.7855e-12, 1.3277e-12, -7.2513e-12, 4.6691e-12, -6.0416e-13,\n -4.3500e-13, -4.8886e-13, -2.2720e-12, -3.2551e-12, 2.9280e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.2640e-12, 2.6123e-12, 4.3654e-13, 1.2327e-11, 1.0220e-13, 1.1338e-12,\n 7.6749e-13, 4.2577e-13, 7.6220e-12, 8.0575e-14, 5.0995e-12, 9.6625e-13,\n 1.3216e-12, 7.7120e-13, 3.4808e-13, 2.5869e-12, 3.7897e-12, 5.5323e-14,\n 1.4157e-12, 1.9777e-12, 9.5104e-13, 9.4754e-13, 1.0063e-12, 6.5147e-13,\n 2.6418e-12, 4.6503e-12, 7.3477e-12, 1.6617e-12, 5.2738e-12, 3.1820e-14,\n 2.1874e-13, 4.4793e-12, 1.5522e-13, 5.6088e-13, 3.2624e-14, 4.3505e-12,\n 6.9171e-13, 2.7116e-12, 5.0772e-13, 8.3968e-14, 2.1417e-12, 4.3702e-12,\n 2.1420e-12, 1.4640e-12, 2.7337e-12, 3.3415e-13, 7.8648e-12, 1.0213e-11,\n 2.8812e-12, 1.6772e-12, 4.6657e-12, 4.0811e-12, 7.2966e-14, 3.4210e-13,\n 4.9522e-12, 2.4405e-12, 3.7222e-13, 9.5033e-13, 1.1443e-13, 3.9315e-12,\n 3.1674e-12, 4.1916e-12, 2.4780e-13, 9.0884e-14, 1.7954e-13, 4.7317e-14,\n 3.9295e-12, 4.7377e-12, 1.4373e-13, 1.9064e-13, 9.0566e-13, 6.0449e-13,\n 2.7870e-12, 2.8601e-12, 8.9742e-13, 2.9180e-12, 3.9234e-13, 8.3718e-13,\n 1.3355e-11, 1.0223e-12, 6.8003e-14, 2.7405e-12, 5.8179e-14, 7.8673e-13,\n 1.8004e-12, 1.9528e-12, 8.9409e-13, 2.8203e-12, 1.5181e-12, 2.2637e-12,\n 5.5250e-12, 1.8861e-12, 5.4088e-12, 1.4918e-12, 3.8173e-13, 2.3970e-13,\n 3.1350e-12, 4.4288e-12, 1.5222e-12, 8.5225e-13, 3.4742e-13, 6.7055e-12,\n 1.4011e-13, 9.2944e-13, 1.0278e-13, 5.9107e-13, 3.3450e-12, 5.5182e-13,\n 6.0100e-13, 5.7217e-14, 6.9245e-13, 7.3844e-14, 2.5113e-13, 2.8402e-12,\n 2.1024e-12, 3.6358e-12, 1.4475e-12, 2.6774e-12, 7.1992e-13, 8.9420e-12,\n 2.5329e-12, 3.2946e-12, 9.9380e-13, 6.0537e-14, 1.3501e-13, 3.2267e-14,\n 1.5023e-12, 2.3883e-12, 1.4893e-12, 1.2261e-13, 8.8733e-14, 2.2752e-12,\n 4.0021e-12, 1.1529e-13, 6.2963e-13, 8.7816e-14, 4.0865e-12, 8.4411e-14,\n 7.7812e-14, 1.3081e-12, 9.1253e-14, 6.0273e-12, 4.0051e-12, 1.3338e-13,\n 4.0798e-12, 3.8225e-12, 6.0701e-13, 1.5541e-12, 1.4436e-12, 8.9604e-13,\n 2.1460e-12, 2.9090e-13, 4.8322e-12, 4.4003e-12, 2.0263e-12, 5.2294e-14,\n 7.1192e-13, 1.1262e-13, 1.0986e-12, 1.3761e-12, 1.1777e-11, 1.6263e-13,\n 1.9740e-12, 2.4603e-13, 3.2766e-12, 1.8957e-12, 1.7178e-13, 1.8060e-13,\n 1.5004e-12, 4.6646e-12, 7.7965e-12, 9.4070e-13, 2.4019e-12, 4.2709e-12,\n 7.4544e-14, 4.2478e-12, 1.6276e-12, 1.2973e-12, 2.9645e-12, 1.2237e-12,\n 8.7197e-13, 9.6932e-13, 4.0666e-12, 1.1206e-12, 8.4193e-12, 8.7893e-12,\n 1.4449e-12, 2.7676e-13, 1.5099e-12, 2.9610e-12, 3.0666e-12, 7.1761e-12,\n 1.3077e-13, 1.7970e-13, 1.9322e-12, 4.1535e-12, 1.6787e-13, 3.6143e-14,\n 8.7809e-14, 2.2487e-12, 1.8015e-12, 2.1229e-13, 7.0022e-13, 1.0050e-13,\n 1.3981e-12, 1.0099e-11, 1.0932e-13, 3.7588e-12, 6.4648e-12, 1.9644e-12,\n 1.9397e-13, 1.2381e-13, 7.7489e-13, 4.4595e-12, 9.6663e-13, 9.9778e-14,\n 2.3637e-12, 7.8893e-13, 1.9159e-13, 2.3734e-12, 1.1077e-12, 2.5964e-12,\n 3.2169e-12, 4.3338e-12, 9.5288e-13, 3.8052e-12, 2.8048e-12, 3.7560e-14,\n 2.4801e-13, 8.7691e-12, 9.7749e-14, 7.2340e-12, 7.7475e-13, 4.6627e-12,\n 2.9692e-12, 1.6683e-13, 5.2300e-12, 1.2217e-12, 1.0433e-11, 3.9574e-14,\n 1.1437e-13, 9.2718e-13, 2.0377e-13, 7.9656e-14, 2.0569e-12, 1.0662e-12,\n 1.4443e-12, 2.7731e-12, 6.8675e-12, 3.6822e-13, 1.6266e-12, 5.1989e-13,\n 6.2744e-12, 4.0049e-12, 4.4367e-13, 6.6256e-13, 2.9783e-13, 5.0875e-14,\n 1.4376e-12, 4.2354e-12, 2.1189e-12, 3.2277e-13, 8.8685e-14, 7.2699e-12,\n 2.1070e-13, 2.0282e-12, 1.0743e-12, 1.4951e-12, 7.3945e-14, 1.8705e-12,\n 4.4691e-14, 1.0118e-12, 1.0780e-13, 2.7519e-12, 1.6967e-13, 5.5010e-12,\n 2.2530e-12, 1.4521e-13, 6.6953e-13, 2.6682e-12, 2.1518e-12, 1.7485e-12,\n 4.0104e-12, 7.1811e-13, 1.6062e-11, 4.2802e-13, 3.3520e-13, 1.0208e-13,\n 6.1995e-12, 1.2507e-13, 8.1834e-13, 1.6978e-12, 7.8961e-13, 1.6902e-13,\n 5.1867e-14, 1.2731e-13, 3.0736e-12, 2.0201e-13, 7.8653e-13, 2.1290e-12,\n 2.3799e-12, 9.7224e-13, 5.6222e-13, 5.4714e-12, 1.5428e-13, 1.6222e-12,\n 1.6934e-13, 1.6906e-13, 1.1366e-13, 6.3575e-14, 1.3704e-12, 1.1275e-12,\n 1.5447e-13, 1.2974e-11, 4.9678e-12, 1.1077e-11, 7.7027e-13, 6.8865e-14,\n 1.9867e-12, 4.2529e-12, 2.7476e-13, 4.4253e-14, 2.0148e-12, 1.6319e-12,\n 3.4313e-13, 1.5511e-12, 1.2665e-13, 2.3814e-13, 4.2908e-12, 4.5565e-12,\n 1.4338e-12, 6.1290e-12, 1.5886e-12, 2.5486e-13, 4.4711e-12, 1.3931e-13,\n 3.1169e-12, 9.6339e-13, 2.4903e-13, 5.6411e-12, 2.5246e-13, 1.9027e-12,\n 4.3499e-12, 1.0452e-12, 3.1228e-12, 1.3822e-12, 6.6731e-12, 2.2653e-13,\n 1.0027e-13, 5.9743e-12, 6.2161e-13, 7.9598e-12, 1.2958e-13, 7.0805e-12,\n 9.6051e-12, 1.5405e-12, 1.0197e-12, 1.3735e-12, 5.8743e-12, 4.2608e-14,\n 3.9035e-12, 1.8028e-12, 3.6603e-12, 3.5352e-13, 9.4905e-13, 9.0169e-13,\n 5.5945e-13, 2.6951e-12, 7.6339e-12, 8.2044e-13, 4.0364e-12, 8.5146e-13,\n 9.2232e-13, 2.0846e-12, 4.5680e-13, 2.2079e-12, 5.4769e-12, 4.3803e-12,\n 1.2189e-12, 7.5082e-12, 1.5103e-12, 1.7364e-13, 1.3421e-12, 5.9234e-13,\n 5.2634e-13, 5.9419e-12, 5.1698e-14, 2.4849e-12, 1.3911e-13, 2.7126e-12,\n 7.5795e-13, 6.7565e-14, 3.2024e-13, 2.0772e-13, 3.6621e-14, 4.3829e-13,\n 4.4972e-12, 7.7092e-12, 4.2868e-14, 3.7644e-13, 7.2364e-12, 6.5754e-13,\n 5.5525e-12, 8.3740e-12, 8.7387e-12, 4.6651e-13, 4.7591e-13, 1.5314e-12,\n 3.0722e-12, 8.5369e-13, 8.6822e-13, 3.8908e-12, 6.1145e-14, 5.9829e-12,\n 9.2313e-13, 7.5684e-13, 3.1442e-14, 2.1199e-12, 1.6910e-12, 4.9646e-13,\n 3.5911e-13, 2.8313e-13, 2.2058e-12, 1.3498e-13, 5.0299e-13, 2.6356e-12,\n 3.4741e-12, 8.2056e-13, 8.2968e-14, 3.3179e-13, 4.2886e-12, 2.8415e-13,\n 2.9769e-13, 5.5102e-12, 4.9068e-13, 2.0516e-12, 4.1233e-12, 4.2313e-12,\n 2.1146e-12, 2.0191e-13, 1.2254e-12, 2.0592e-13, 6.7392e-12, 5.2322e-14,\n 2.2809e-12, 3.8299e-13, 1.0641e-13, 7.3979e-14, 1.4368e-11, 1.1165e-11,\n 1.6054e-13, 1.2740e-13, 8.3303e-12, 3.3268e-13, 3.3265e-12, 3.0168e-13,\n 3.3940e-12, 6.5617e-12, 5.9937e-12, 1.6680e-12, 1.3067e-11, 8.6153e-12,\n 8.0006e-13, 2.7195e-13, 1.2746e-13, 7.9267e-12, 1.3676e-13, 4.3743e-12,\n 6.7302e-13, 8.9450e-14, 6.3053e-13, 1.7212e-13, 1.3859e-12, 3.2305e-12,\n 5.1978e-13, 4.0268e-14, 7.6963e-13, 2.3261e-12, 1.4642e-12, 5.4572e-13,\n 3.6810e-13, 5.3694e-12, 1.5058e-12, 1.9749e-12, 7.6835e-13, 3.5343e-13,\n 2.5384e-12, 6.4630e-13, 1.6316e-12, 2.0530e-13, 1.6205e-11, 4.0530e-12,\n 1.9005e-13, 1.2580e-12, 1.2288e-12, 4.8089e-13, 1.3675e-13, 6.5131e-13,\n 1.9782e-12, 3.9953e-13, 1.5768e-12, 2.7810e-13, 2.0628e-12, 5.8002e-12,\n 6.9369e-12, 2.8010e-12, 1.6944e-13, 1.5808e-12, 1.1481e-13, 2.4115e-13,\n 3.9360e-13, 1.2401e-12, 1.0367e-13, 2.7482e-12, 1.2667e-11, 9.5918e-12,\n 2.0290e-12, 5.9193e-14, 1.1323e-13, 4.0282e-12, 1.3395e-12, 2.4066e-13,\n 2.0560e-13, 1.1000e-12, 5.7681e-13, 6.8521e-12, 3.2703e-13, 4.5380e-12,\n 2.3381e-12, 3.4201e-12, 2.4711e-13, 3.3285e-12, 9.0290e-12, 1.1721e-13,\n 4.0363e-12, 5.1448e-13, 8.9784e-14, 2.6138e-13, 1.7842e-12, 3.0469e-12,\n 1.5591e-12, 3.1480e-13, 7.4834e-13, 2.0013e-13, 2.7450e-13, 1.3718e-12,\n 3.3809e-12, 1.5540e-12, 1.4054e-13, 1.2228e-12, 1.2964e-12, 7.5495e-13,\n 1.9844e-14, 1.0030e-12, 2.1030e-12, 9.3984e-14, 1.7229e-13, 8.8419e-14,\n 9.0979e-12, 2.4976e-13, 1.6985e-11, 3.1280e-12, 5.8356e-14, 8.2785e-12,\n 1.0778e-13, 1.2605e-13, 6.4694e-12, 1.8687e-12, 5.3992e-13, 8.7085e-13,\n 4.2208e-13, 7.2956e-13, 1.1244e-11, 6.7032e-13, 3.6156e-12, 1.0817e-12,\n 3.0276e-12, 5.6839e-13, 6.3180e-13, 2.1135e-12, 4.5020e-13, 1.8765e-12,\n 5.6251e-12, 1.2311e-13, 1.5141e-12, 2.9322e-12, 3.0263e-13, 3.6767e-12,\n 2.6704e-12, 1.4432e-12, 1.0215e-12, 7.9351e-13, 1.0359e-12, 1.2657e-13,\n 4.2426e-12, 3.4453e-12, 3.6116e-12, 4.8161e-12, 5.1175e-13, 3.9922e-12,\n 2.6126e-12, 2.7560e-12, 5.2556e-13, 1.3096e-12, 8.6943e-13, 4.0689e-14,\n 2.3444e-12, 1.1573e-13, 1.6401e-12, 1.9395e-12, 2.9873e-12, 8.3767e-12,\n 3.6706e-12, 3.9322e-14, 2.4697e-13, 6.0376e-13, 4.6984e-13, 1.5184e-12,\n 6.7222e-12, 1.0534e-12, 2.4180e-12, 5.8676e-14, 2.2342e-12, 3.3642e-12,\n 1.5264e-13, 4.2166e-12, 8.3383e-14, 1.3316e-12, 1.5103e-13, 1.7852e-12,\n 7.9976e-12, 9.2244e-15, 7.5170e-13, 3.8651e-12, 3.1483e-14, 2.3580e-13,\n 3.5237e-14, 4.7030e-12, 3.3661e-12, 7.3147e-12], device='cuda:0')" + }, + "40": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.8925e-14, 3.9016e-14, 1.1595e-14, -1.6940e-15, -5.6455e-15,\n -5.6032e-16, -5.4194e-15, 2.8252e-13, -9.9200e-15, -4.4896e-15,\n 3.4760e-14, -1.4630e-15, 4.8610e-14, 8.9809e-16, 3.0458e-15,\n -1.0503e-14, 9.5807e-14, -4.8603e-15, 2.6748e-14, -9.3104e-15,\n -9.3379e-15, 1.6551e-14, 1.1355e-15, -3.5964e-15, 3.0377e-13,\n 1.2107e-13, 3.9562e-13, -3.9565e-15, 1.4259e-13, -1.7618e-15,\n 1.9172e-15, -4.1854e-15, 2.8156e-15, 2.5815e-14, -9.9765e-15,\n 8.3720e-16, 1.5694e-14, -1.7273e-15, -3.0122e-15, -7.0481e-15,\n 1.3291e-13, 8.8541e-14, -1.1119e-15, 2.8348e-15, 9.1271e-14,\n -7.1556e-15, 3.1692e-14, 2.2370e-14, 3.0272e-14, 1.8181e-14,\n 6.9607e-14, 3.4590e-14, 3.3108e-14, 2.3502e-14, -3.3361e-15,\n 2.0693e-13, 2.4686e-14, 1.0250e-14, -1.8251e-18, 5.4863e-14,\n -3.0741e-14, 1.1790e-15, 1.7505e-15, 2.1492e-14, -1.0470e-14,\n -6.7675e-16, 5.1789e-14, 2.1266e-15, 5.5390e-15, 3.3711e-14,\n -4.3098e-15, -8.1518e-15, -1.5774e-15, -3.4120e-15, 5.5729e-14,\n 7.3863e-14, -9.6472e-15, 9.6180e-14, 1.3754e-13, 6.4885e-14,\n 1.3442e-14, 7.4068e-14, -2.5767e-14, 1.4753e-13, 2.8060e-14,\n 1.1752e-14, 4.0225e-14, -1.3581e-14, 2.0776e-14, 2.3111e-14,\n 5.4328e-14, -8.2370e-15, 1.1244e-14, -2.0237e-14, -1.1388e-14,\n 4.9214e-14, 1.0006e-15, 2.5321e-15, 1.0656e-14, 1.0892e-13,\n 6.2950e-16, 1.2390e-13, 2.4178e-13, 6.3411e-14, 1.8918e-14,\n -2.5505e-15, 8.4564e-16, 2.6958e-14, -1.2933e-14, 6.4016e-15,\n 2.3282e-14, 7.9405e-15, 7.6131e-15, -8.8396e-16, 8.8638e-14,\n 1.9261e-15, -1.4327e-15, 2.5659e-13, 9.8278e-15, 1.7623e-13,\n -1.0845e-14, -6.2490e-15, 5.4363e-14, 5.3427e-15, -1.1733e-14,\n -4.5289e-15, 2.4500e-15, 1.0033e-15, 1.6052e-14, 9.7672e-15,\n 1.6123e-14, 1.6470e-14, 1.2447e-13, 1.1431e-14, -8.3358e-15,\n -9.4236e-15, 1.6605e-14, -4.1913e-16, 2.5163e-14, 1.8930e-14,\n -1.5009e-14, 4.4866e-13, -2.5025e-14, 2.5617e-14, -4.8476e-15,\n 3.9739e-14, 2.8417e-14, 6.3503e-15, 6.3270e-14, 3.0788e-14,\n 4.3951e-14, 3.9990e-15, 1.2127e-13, 7.0748e-15, 3.1368e-13,\n -9.2453e-15, 1.6690e-13, -5.1988e-15, -2.6578e-15, -1.1806e-15,\n 1.3899e-13, -6.7801e-15, 2.4000e-13, -8.8303e-15, 1.5926e-15,\n -1.6412e-14, 2.0759e-14, -4.0190e-15, 6.9836e-14, 1.9409e-14,\n 3.4104e-13, -4.0518e-15, -9.1594e-16, 7.7248e-16, -2.8216e-15,\n 7.6484e-15, 1.7019e-14, 2.6659e-14, 3.3517e-14, 8.7774e-14,\n 1.0553e-13, -1.0399e-14, -5.8858e-15, -1.4930e-14, 8.1789e-14,\n 1.7439e-13, 1.9314e-13, -1.3089e-15, -2.1825e-16, 5.3994e-15,\n -5.2947e-16, 1.3577e-13, 4.4664e-15, 3.2265e-16, -9.1006e-16,\n 1.2489e-15, -4.0621e-15, 9.2930e-15, -8.0739e-16, -8.2679e-15,\n 2.0804e-13, -5.8485e-16, -1.2913e-14, 5.1020e-16, 1.2338e-13,\n -1.9276e-13, 2.3168e-14, 6.8959e-15, 1.3923e-13, 6.7305e-15,\n 3.0929e-14, 1.6427e-14, 6.8399e-14, 6.6350e-14, -2.7738e-15,\n 1.4779e-14, 1.0652e-15, 6.9638e-14, 1.4273e-14, 1.4262e-14,\n 1.1651e-14, -1.3933e-14, 6.0580e-14, -3.5231e-15, 1.1361e-15,\n 1.7457e-13, -3.8853e-15, 4.2068e-15, 1.2614e-14, 1.1827e-13,\n 1.8741e-14, 7.1328e-14, 7.8690e-17, 1.9865e-13, 7.1164e-14,\n 1.5103e-15, 1.4480e-13, -2.1001e-14, 1.3678e-13, 2.4978e-14,\n 3.7637e-14, -3.0489e-15, 7.4060e-16, 6.5819e-15, 1.6322e-14,\n 1.5677e-14, 9.9100e-14, 2.1557e-13, 9.8199e-14, -7.1461e-15,\n 3.4258e-14, -5.7766e-15, 4.1879e-13, 7.4844e-15, 1.3816e-14,\n 3.6388e-14, -1.8847e-15, 4.5560e-15, 8.1663e-14, 1.1300e-13,\n -6.1250e-16, -5.2098e-15, 1.1091e-14, 5.4750e-15, 1.6206e-14,\n -2.3435e-15, -3.4038e-15, 1.3068e-15, -5.3006e-16, 5.9658e-14,\n 7.4013e-14, 1.6437e-13, 1.4232e-14, -1.3686e-14, 4.2611e-15,\n -9.0464e-15, -3.1609e-15, 2.0231e-14, -1.0966e-14, -4.8880e-15,\n -1.5576e-14, 1.0785e-13, 2.6839e-14, 2.2424e-16, -1.2532e-14,\n -1.3454e-14, -5.5328e-15, 1.8529e-15, -2.5818e-13, -1.0226e-14,\n 9.4794e-14, -3.3831e-15, 7.5555e-15, 1.1320e-14, -1.8741e-15,\n -1.0793e-14, -3.6785e-15, -6.6664e-14, -2.1214e-16, 7.4397e-14,\n 5.1001e-15, -3.5412e-15, -2.0659e-14, -7.7340e-15, 5.3946e-16,\n -8.4624e-15, 4.7160e-16, -1.2842e-14, 1.5566e-14, -5.9145e-15,\n -7.1088e-15, 6.2355e-13, 6.2606e-14, 3.8055e-13, 3.9857e-14,\n -1.6094e-15, 3.4043e-14, 1.3731e-14, -1.5110e-14, 7.8268e-14,\n -2.3325e-15, 1.3937e-13, -1.3557e-16, 1.1261e-13, 1.8184e-14,\n -7.7650e-15, -1.3715e-14, 2.9171e-14, 5.0756e-15, 2.2852e-13,\n 3.5094e-15, 1.1463e-13, 1.8500e-15, 3.4705e-17, -3.3700e-14,\n -1.7071e-15, 1.5380e-15, 5.3961e-14, -1.1504e-14, 1.0334e-13,\n -1.6302e-14, 3.6018e-15, 2.4262e-14, -3.5337e-15, -1.8277e-14,\n -3.5550e-14, 3.4394e-13, -3.6695e-15, 3.7968e-15, 1.2265e-14,\n -1.3602e-15, 1.0295e-13, 2.8243e-15, 1.2651e-13, 9.8249e-14,\n -1.1037e-14, 1.7557e-13, 9.1266e-17, 8.4311e-14, -1.0886e-15,\n -2.0483e-15, 1.6400e-14, -1.8178e-15, 2.5761e-15, -6.3338e-15,\n -1.1580e-14, -1.4257e-16, -4.9044e-15, -4.8061e-16, 3.2788e-15,\n -4.4579e-15, 1.6000e-15, 2.6226e-14, -8.8120e-15, -3.0610e-15,\n 1.3788e-14, 6.8244e-15, -1.8581e-14, -6.5240e-15, 1.8927e-14,\n -2.6387e-16, 6.8734e-14, 3.4737e-14, 1.6055e-14, 2.0720e-13,\n -4.4221e-17, -4.2840e-15, 4.3027e-14, 2.4907e-14, -5.8772e-15,\n 7.1379e-16, 1.0075e-14, -3.1298e-15, 2.8290e-15, -4.1234e-15,\n -1.1372e-14, -8.4710e-15, 4.1434e-15, 6.5337e-15, -7.1294e-15,\n 1.4796e-14, -1.2997e-15, 4.8416e-14, 2.5372e-14, 1.6775e-13,\n -7.7635e-15, -1.1795e-14, -1.5785e-15, 1.5228e-14, -7.5831e-15,\n 2.2032e-14, 1.9572e-13, 7.7881e-15, -7.0427e-17, 6.6274e-15,\n 4.2672e-14, -7.6640e-15, 6.0493e-14, -7.1796e-15, 4.5563e-15,\n 1.9175e-15, -8.2921e-15, 9.8268e-14, 5.2734e-15, 5.2999e-15,\n 2.0729e-13, 1.8670e-13, -1.0299e-14, 1.6146e-14, -1.3617e-15,\n 9.4659e-14, 1.1333e-14, -5.8059e-15, 3.5424e-15, -2.2159e-15,\n 1.1112e-13, 9.6763e-14, -1.4762e-14, 2.6241e-13, 3.0659e-16,\n -7.7679e-15, 9.5870e-17, 7.6055e-15, -2.6531e-16, -1.2612e-14,\n 7.4638e-14, 1.1752e-16, 9.6537e-15, 1.6591e-13, 3.4577e-15,\n 4.6618e-15, 1.7046e-14, 6.6873e-14, 2.9148e-16, 2.5778e-14,\n -3.4767e-15, 2.4095e-14, 1.2967e-14, 1.9921e-13, 2.6498e-14,\n 1.8961e-13, 5.4980e-14, 9.8524e-15, 2.9407e-14, -5.8981e-15,\n -1.4520e-14, -8.8337e-15, 8.3976e-14, -2.8000e-15, 9.5197e-16,\n 1.8826e-14, 3.3122e-13, -1.4517e-15, 5.7139e-14, -3.0594e-15,\n 1.2518e-14, -8.1301e-15, -1.5399e-15, 3.0170e-14, 1.1956e-14,\n 1.7358e-14, 2.1163e-15, 2.1899e-13, -9.3114e-14, -1.9933e-15,\n 1.5277e-14, -1.2288e-14, -3.0560e-15, 1.4280e-13, 6.4848e-14,\n 3.3535e-13, 9.3322e-14, -1.4036e-15, -8.8129e-15, 9.5753e-14,\n 4.5942e-14, 1.8534e-16, -6.9083e-15, -1.3746e-13, 1.4692e-14,\n 1.1637e-15, -3.6344e-15, 2.2725e-14, -3.1698e-15, 2.5802e-14,\n 8.0266e-14, 1.1115e-15, 7.2016e-15, -2.5907e-15, 5.4702e-14,\n 1.3998e-15, 8.2807e-14, 1.6300e-14, 7.0823e-15, 1.1287e-13,\n 3.7398e-15, 1.0650e-13, -1.1705e-14, -9.8045e-15, 5.8110e-14,\n -1.6426e-15, -6.1258e-15, 4.9717e-15, 1.4212e-14, -5.2326e-15,\n 3.9840e-14, -2.9644e-14, 4.0046e-14, 5.0586e-14, -1.2909e-15,\n 1.8348e-14, 1.0181e-13, 8.5040e-15, 9.0922e-15, 2.0881e-13,\n 8.5560e-14, 1.2349e-15, 1.9112e-15, 6.0647e-15, 8.5141e-14,\n -1.1093e-14, 2.0335e-15, 5.7962e-14, -8.9619e-15, -3.1776e-15,\n -1.4814e-15, 5.2618e-14, 5.8256e-14, 2.0908e-14, 2.8129e-14,\n 1.0961e-13, -1.7915e-14, 7.4557e-14, -2.1758e-15, -4.1240e-14,\n -7.0592e-16, 4.5411e-15, 2.6488e-14, 2.0843e-13, -8.1533e-15,\n 3.7883e-13, 3.4974e-14, -3.4326e-15, 2.9047e-13, 2.4096e-15,\n 1.6933e-15, 2.0054e-13, 1.4410e-14, 5.8628e-14, -1.3495e-14,\n -2.7304e-15, 3.9800e-14, -2.7180e-14, -5.9328e-15, -9.4753e-18,\n 2.2952e-15, -4.9294e-16, 5.7164e-15, 1.2091e-15, 2.3891e-14,\n -4.3539e-15, -1.7749e-14, 7.6221e-14, -3.0782e-15, 6.1182e-15,\n 2.9581e-14, 1.5168e-14, -8.3046e-16, -7.3796e-15, 6.0875e-17,\n 3.1632e-14, 3.4584e-14, -4.6578e-15, 2.8204e-16, 1.6307e-13,\n 1.1622e-13, 1.4651e-15, 4.6494e-14, 2.1540e-13, 3.1999e-14,\n 2.3170e-13, 1.6479e-14, -4.9329e-15, 1.3659e-14, 2.8375e-16,\n -5.1794e-15, -1.0029e-14, 5.3609e-15, 1.3887e-14, 1.4674e-14,\n 2.7230e-14, -6.1257e-15, 4.1134e-14, 1.1669e-13, -1.4141e-17,\n -1.4284e-15, -1.6635e-15, 8.1804e-15, 1.1795e-13, 2.2281e-13,\n 4.9821e-14, -1.2873e-14, 5.9170e-15, -2.5024e-15, 6.8345e-15,\n -1.3271e-13, 1.4454e-13, 1.5864e-14, -2.3664e-14, 7.5070e-14,\n 1.9654e-13, -2.2339e-15, -1.2554e-13, 8.5993e-14, 3.5892e-14,\n -8.7354e-15, 1.6409e-14, -3.0661e-15, -4.4458e-15, 1.0642e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.4643e-16, 3.3901e-16, 2.0486e-17, 5.9822e-15, 7.1786e-18, 1.1821e-16,\n 1.3036e-15, 1.7410e-16, 3.2325e-15, 3.9209e-17, 2.9488e-16, 5.4209e-16,\n 7.6474e-16, 5.4109e-16, 2.0125e-16, 4.9649e-16, 1.7778e-15, 9.3943e-17,\n 3.9640e-16, 9.0704e-15, 6.4532e-16, 3.7685e-16, 1.2578e-16, 1.4971e-17,\n 7.0764e-16, 4.2844e-15, 6.1936e-15, 2.5746e-16, 2.2182e-15, 7.9588e-18,\n 6.2430e-17, 4.7339e-16, 8.6253e-17, 8.7241e-17, 2.2375e-17, 1.4724e-15,\n 2.3700e-17, 1.2419e-16, 3.5045e-17, 9.9883e-18, 1.4902e-15, 3.3908e-15,\n 1.3895e-15, 1.0079e-15, 3.0657e-15, 5.6931e-18, 2.0271e-15, 6.3528e-15,\n 4.5556e-16, 1.4045e-15, 2.1842e-15, 1.3339e-14, 3.9761e-17, 7.3763e-18,\n 2.2827e-16, 7.9298e-16, 1.5577e-16, 7.8346e-17, 5.1626e-17, 2.2763e-15,\n 2.1424e-15, 1.7931e-15, 3.6557e-17, 2.1055e-16, 1.1974e-16, 7.0538e-17,\n 3.9311e-15, 1.4530e-15, 9.7276e-18, 3.6470e-17, 2.1280e-16, 9.9744e-17,\n 6.0340e-16, 1.5814e-15, 1.0096e-16, 7.2389e-16, 4.4266e-17, 2.1530e-16,\n 2.7298e-14, 2.0412e-16, 6.6490e-17, 4.4318e-16, 5.9834e-18, 1.5485e-16,\n 2.2816e-17, 6.5101e-17, 7.7009e-17, 5.6440e-16, 5.3035e-16, 8.4703e-15,\n 6.5038e-15, 1.0981e-15, 7.0528e-15, 1.3626e-16, 7.6301e-17, 1.4630e-17,\n 1.6465e-15, 4.9219e-15, 2.8634e-16, 3.4209e-16, 4.9795e-18, 3.3561e-15,\n 1.6158e-17, 4.3017e-15, 7.9898e-17, 2.9874e-17, 1.3155e-15, 2.9036e-17,\n 2.7617e-17, 3.0311e-18, 2.5221e-17, 5.2667e-17, 2.1630e-17, 1.9263e-15,\n 1.3953e-15, 2.5639e-15, 1.1349e-15, 8.0199e-16, 6.0108e-17, 5.9404e-15,\n 7.7266e-16, 6.0211e-15, 4.6569e-17, 6.0712e-18, 5.9965e-18, 8.0290e-17,\n 1.0808e-16, 5.8988e-15, 3.2527e-16, 5.8904e-17, 2.2108e-17, 9.9462e-16,\n 8.5519e-16, 6.5398e-17, 5.6779e-17, 1.0017e-16, 2.3193e-15, 4.7978e-17,\n 5.9549e-17, 1.9832e-16, 3.6251e-17, 4.4256e-15, 3.8893e-15, 5.1769e-17,\n 3.5837e-15, 7.9445e-16, 4.5794e-17, 3.7298e-17, 8.7458e-16, 2.1332e-15,\n 4.0948e-15, 3.8917e-17, 1.0367e-14, 3.3192e-16, 9.8074e-16, 3.1372e-17,\n 1.9424e-16, 4.7528e-17, 7.4537e-17, 3.4496e-16, 1.1627e-14, 4.4167e-17,\n 1.9958e-15, 9.1662e-18, 2.6897e-16, 6.5689e-16, 5.5162e-17, 6.0508e-17,\n 4.5780e-17, 5.1002e-15, 1.2497e-14, 1.1999e-17, 2.6889e-16, 6.7896e-15,\n 5.8883e-17, 1.4077e-15, 1.3860e-15, 1.9327e-17, 8.0855e-16, 1.0266e-16,\n 1.8285e-15, 9.0901e-17, 1.0199e-15, 6.0638e-17, 1.4602e-14, 1.0752e-14,\n 2.3896e-15, 5.7914e-18, 2.6673e-15, 4.9521e-16, 5.8110e-16, 1.1782e-15,\n 1.3471e-17, 6.9832e-17, 1.4083e-15, 1.4595e-15, 1.1835e-16, 6.8200e-17,\n 2.2215e-16, 1.1414e-16, 1.3060e-15, 9.5683e-18, 7.1525e-16, 3.1623e-17,\n 9.5508e-16, 2.4203e-14, 6.3172e-17, 4.1989e-15, 5.3231e-15, 6.9400e-15,\n 7.7217e-17, 4.2019e-17, 1.9339e-16, 5.6381e-15, 4.6588e-16, 1.1512e-16,\n 2.7941e-16, 2.8084e-16, 2.4378e-17, 1.5254e-15, 5.6022e-16, 2.0620e-15,\n 6.8136e-16, 1.0152e-15, 1.5312e-16, 5.1972e-15, 2.0838e-15, 8.5986e-17,\n 5.4650e-18, 4.5976e-15, 2.8845e-17, 3.4231e-15, 1.1318e-16, 5.2604e-15,\n 3.4575e-15, 7.1638e-18, 3.7011e-15, 1.0113e-15, 9.8477e-15, 3.6862e-17,\n 4.9828e-17, 7.8651e-16, 2.3946e-16, 9.6635e-17, 6.7847e-16, 1.3163e-15,\n 7.4675e-17, 8.5970e-16, 1.3195e-14, 3.4340e-17, 1.7119e-15, 1.1916e-17,\n 6.3857e-15, 2.2204e-15, 4.5517e-18, 2.5460e-17, 2.0522e-17, 4.9495e-18,\n 6.5848e-16, 4.6450e-15, 3.5841e-15, 1.3682e-17, 2.3237e-17, 7.0744e-16,\n 5.8244e-18, 1.1268e-15, 3.1818e-16, 5.2364e-16, 6.3788e-18, 1.5982e-15,\n 5.1376e-17, 9.0589e-16, 4.5130e-17, 1.5975e-15, 4.8501e-17, 3.4951e-15,\n 9.5906e-16, 5.6862e-17, 1.4538e-16, 3.8306e-16, 1.4450e-15, 1.4673e-15,\n 6.0573e-15, 1.5119e-16, 4.7685e-15, 6.3749e-18, 4.7999e-17, 1.1709e-16,\n 3.8091e-15, 1.7421e-17, 1.7870e-17, 5.5474e-16, 1.1650e-16, 1.0362e-16,\n 4.3022e-17, 6.7910e-17, 1.5716e-16, 1.0720e-16, 3.6363e-17, 1.2189e-15,\n 1.6501e-15, 1.1515e-16, 8.1239e-18, 5.7701e-16, 2.4934e-17, 8.8463e-16,\n 3.3438e-16, 1.6847e-16, 1.0631e-16, 5.1425e-18, 2.3750e-16, 4.6458e-15,\n 5.1330e-17, 9.1825e-15, 5.8487e-15, 8.2951e-15, 9.1868e-18, 4.2268e-17,\n 5.3860e-15, 3.5142e-15, 2.1262e-17, 1.0787e-17, 4.2647e-16, 3.6573e-16,\n 5.0963e-17, 3.3975e-16, 5.2226e-17, 1.6861e-16, 1.0925e-15, 5.0974e-15,\n 2.3749e-16, 2.1734e-15, 3.7315e-17, 2.5518e-17, 4.8392e-16, 1.3617e-17,\n 3.4099e-16, 1.4832e-16, 2.3250e-17, 3.1636e-15, 7.0426e-17, 6.0051e-16,\n 1.6472e-15, 5.2944e-17, 6.4538e-15, 1.9126e-16, 8.4706e-15, 1.8516e-17,\n 7.8102e-17, 5.3769e-15, 2.7933e-17, 3.8996e-15, 7.9517e-17, 1.4136e-15,\n 5.3738e-15, 1.0521e-16, 2.7553e-16, 1.2908e-15, 2.4035e-14, 1.7866e-17,\n 3.7223e-15, 1.7793e-15, 1.3829e-15, 2.7386e-17, 2.6084e-16, 5.0731e-16,\n 2.6692e-17, 1.2512e-15, 3.6731e-15, 2.0314e-16, 8.3421e-15, 7.0752e-17,\n 2.9854e-17, 5.6638e-16, 7.0477e-18, 2.4379e-16, 1.6125e-15, 7.7416e-16,\n 1.6389e-17, 1.1276e-14, 5.1666e-16, 1.1250e-16, 2.4302e-16, 4.3280e-17,\n 1.1540e-16, 6.0509e-16, 5.8414e-17, 3.1689e-15, 9.6456e-17, 2.0370e-15,\n 7.8738e-17, 3.3260e-18, 7.9734e-17, 1.8486e-17, 8.9254e-18, 2.8645e-17,\n 1.9433e-15, 8.6770e-15, 4.2692e-17, 9.8097e-18, 4.2826e-15, 5.6742e-17,\n 1.5228e-14, 1.9841e-14, 1.4570e-14, 6.3338e-17, 2.5212e-17, 4.1985e-16,\n 2.4906e-16, 2.6109e-17, 8.1604e-16, 4.5633e-15, 1.9413e-17, 9.8685e-16,\n 1.5683e-16, 3.6806e-16, 4.5138e-18, 4.7858e-16, 4.7301e-17, 1.0772e-16,\n 6.1758e-18, 2.6721e-17, 5.1949e-15, 6.5596e-18, 9.6037e-17, 4.8878e-16,\n 8.8842e-15, 5.2283e-17, 8.2613e-17, 3.6159e-17, 4.9673e-15, 3.7644e-17,\n 2.1202e-17, 3.3937e-16, 3.7703e-17, 1.8007e-16, 2.7654e-14, 1.9775e-15,\n 1.6225e-15, 8.5122e-18, 1.2350e-16, 2.2977e-17, 2.4570e-14, 9.8516e-18,\n 3.0588e-16, 1.5524e-17, 2.3845e-16, 7.6157e-18, 2.6397e-14, 7.1803e-15,\n 4.9892e-18, 2.7100e-17, 9.4276e-15, 3.9363e-17, 5.1980e-15, 2.0656e-18,\n 3.1560e-15, 7.8401e-15, 1.7854e-15, 7.6917e-17, 1.5467e-14, 2.3066e-15,\n 4.7207e-16, 8.1376e-18, 9.2665e-17, 3.2883e-15, 6.6695e-17, 1.0875e-15,\n 2.4643e-16, 4.4349e-17, 1.0697e-16, 2.0271e-16, 1.1069e-16, 1.3399e-15,\n 4.5357e-17, 3.2372e-18, 2.8075e-16, 3.2613e-16, 3.2274e-17, 1.2545e-17,\n 9.4460e-18, 1.9614e-15, 8.4535e-16, 6.7462e-17, 1.2496e-17, 3.5695e-17,\n 3.3021e-16, 1.6485e-16, 3.4564e-16, 7.3688e-18, 2.1059e-14, 5.9160e-16,\n 2.2760e-18, 1.0385e-16, 1.9015e-16, 1.1470e-16, 5.3639e-17, 3.9346e-16,\n 6.1104e-16, 3.4006e-17, 4.7644e-16, 6.0473e-18, 1.0842e-15, 1.5450e-15,\n 1.2755e-14, 1.8157e-15, 7.5456e-18, 1.7119e-16, 2.1125e-17, 9.5902e-18,\n 1.4313e-17, 2.8236e-16, 3.9898e-18, 2.2823e-16, 1.1538e-14, 8.3422e-15,\n 7.3699e-16, 1.1076e-16, 6.5664e-18, 2.1639e-15, 3.7740e-17, 2.0682e-17,\n 1.4760e-16, 2.2160e-16, 6.1573e-18, 5.7128e-15, 5.7602e-18, 4.1793e-16,\n 7.1946e-15, 1.6257e-15, 9.0282e-17, 2.2084e-15, 5.5666e-15, 4.8408e-17,\n 1.4552e-15, 2.0329e-17, 3.2719e-17, 4.8470e-17, 1.7704e-15, 1.9739e-15,\n 4.7025e-16, 1.5943e-17, 1.1791e-16, 5.5881e-18, 5.3670e-18, 4.5087e-16,\n 4.7384e-15, 4.3722e-16, 4.4631e-17, 6.9933e-17, 6.6774e-16, 3.4982e-16,\n 3.0951e-17, 8.1484e-16, 1.4527e-14, 1.0303e-17, 9.3794e-17, 7.2769e-17,\n 8.6667e-15, 1.4729e-17, 2.4013e-14, 1.7220e-15, 5.2436e-17, 2.6185e-14,\n 7.5003e-18, 1.2066e-16, 5.0098e-15, 2.6299e-15, 8.5821e-17, 4.7365e-17,\n 9.8242e-17, 2.4099e-17, 8.7020e-15, 1.3497e-16, 1.7238e-15, 1.1151e-15,\n 1.5054e-15, 1.1223e-17, 6.9304e-17, 4.8546e-15, 5.4713e-17, 8.6565e-17,\n 2.4175e-15, 3.1509e-18, 1.0790e-15, 1.6390e-15, 5.5506e-17, 8.8884e-16,\n 1.4572e-16, 6.0935e-16, 2.1567e-17, 5.0151e-17, 2.8547e-17, 4.3768e-17,\n 7.8358e-15, 3.6550e-15, 6.5350e-15, 6.8596e-15, 1.1821e-16, 3.5657e-15,\n 5.1624e-15, 4.6695e-16, 2.0554e-16, 3.5524e-16, 5.2196e-17, 2.2018e-17,\n 5.7625e-16, 4.3481e-17, 5.2161e-16, 6.3785e-16, 1.2621e-15, 9.9851e-15,\n 2.3799e-16, 7.5606e-18, 5.1007e-17, 5.9081e-17, 4.0465e-18, 5.5423e-16,\n 9.1013e-15, 1.0298e-15, 3.3512e-15, 1.7853e-17, 8.3888e-16, 1.1774e-15,\n 1.0969e-17, 6.4754e-15, 1.5533e-17, 1.2844e-16, 1.2610e-16, 5.1750e-16,\n 5.9997e-15, 2.5131e-18, 7.8849e-16, 3.2657e-15, 9.2148e-18, 2.1777e-16,\n 7.3504e-18, 8.4210e-16, 1.8850e-15, 4.4278e-15], device='cuda:0')" + }, + "41": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 4.0420e-14, 4.6927e-14, -1.2837e-14, 9.0006e-14, 2.4395e-14,\n 2.2959e-14, -3.3177e-15, 1.6623e-13, 2.0286e-14, 6.3704e-15,\n -2.9169e-14, -1.6592e-15, 1.8225e-13, -1.7220e-15, -3.6842e-15,\n 3.0305e-14, 1.3521e-13, 9.4866e-15, 8.8899e-14, -1.1201e-14,\n 3.2251e-14, 5.8994e-14, -5.1714e-15, 2.2328e-14, 3.0505e-13,\n 1.3450e-13, 1.7752e-13, 3.0540e-15, 1.8433e-13, 2.4438e-15,\n -1.9666e-15, 6.8384e-15, -1.4794e-15, -2.0322e-14, 1.4244e-14,\n -3.2768e-15, -1.2747e-14, 2.4518e-15, 2.4576e-15, 9.9587e-14,\n 1.1629e-13, 1.4410e-13, 1.6735e-15, 1.1404e-14, 1.3727e-13,\n 2.2097e-14, 1.2808e-13, 4.2331e-14, 1.1875e-13, 4.1321e-14,\n 1.3972e-13, 2.8360e-14, -2.7644e-14, 3.1790e-14, 9.8057e-14,\n 1.8448e-13, 5.4985e-14, 1.5488e-13, 6.9894e-14, 1.8199e-13,\n -3.4581e-14, 7.9485e-14, -5.3165e-16, -1.0109e-14, 4.0084e-14,\n 5.0580e-16, 3.8535e-14, 1.1371e-15, -6.2227e-15, -1.7182e-14,\n 4.5306e-15, 1.1859e-14, 5.1924e-14, 6.0602e-15, 1.2008e-13,\n 4.8301e-14, 2.8772e-14, 1.1167e-13, 1.0572e-13, 5.4867e-14,\n -7.1746e-15, 1.3835e-13, 1.0275e-13, 2.1319e-13, -2.7053e-14,\n 2.2227e-14, 7.4690e-14, 4.9310e-14, 1.9409e-14, 6.3238e-14,\n 8.9264e-14, 1.7954e-14, 8.9774e-15, -5.0959e-14, 3.6671e-14,\n 5.1157e-14, 1.5094e-14, 5.2956e-15, 2.4346e-14, 1.2528e-13,\n -9.7655e-15, 7.2395e-14, 2.9573e-13, 9.3333e-14, -1.7959e-14,\n -1.6418e-14, 3.9104e-14, 6.8433e-14, 4.3461e-14, -5.9794e-15,\n 5.3727e-14, -1.3079e-14, -7.3277e-15, 6.0633e-15, 1.5575e-13,\n -3.5764e-15, 2.1016e-14, 1.9850e-13, 1.8860e-14, 1.5891e-13,\n 2.7451e-14, -6.5319e-15, 1.6525e-13, 3.3188e-14, 4.8404e-14,\n 2.6412e-15, -2.6108e-15, -1.0434e-14, 6.4925e-14, -5.2834e-15,\n 7.8339e-14, 1.8610e-14, 1.7866e-13, -5.4336e-15, 7.8121e-14,\n 1.6185e-14, 1.1381e-13, 2.2648e-16, -1.5514e-14, -1.4131e-14,\n 6.0503e-14, 2.7640e-13, -2.5428e-14, -1.7262e-14, 7.8858e-15,\n 7.9539e-14, 9.4493e-14, 3.7071e-14, 2.2412e-13, 8.3236e-14,\n 8.6496e-14, -3.6542e-15, 2.1867e-13, 8.8587e-14, 2.7440e-13,\n 4.2511e-14, 1.9248e-13, 2.2302e-14, 9.9772e-15, 3.9042e-14,\n 1.3847e-13, 1.4712e-14, 1.9554e-13, 1.9360e-14, 3.1929e-14,\n 9.5038e-14, 1.0915e-13, 1.8382e-14, 1.4672e-13, 1.0713e-13,\n 3.3792e-13, 7.0069e-14, -4.9325e-14, -9.9950e-15, 3.1927e-15,\n 2.7313e-14, 7.7214e-14, 3.3549e-14, 5.5012e-14, 6.5072e-14,\n 1.4546e-13, 8.8357e-15, 8.4603e-14, 3.4830e-14, 5.5319e-14,\n 1.2443e-13, 1.3761e-13, 4.1680e-15, 3.4699e-14, 3.7170e-15,\n 6.6229e-15, 1.4750e-13, 8.0780e-14, -1.1710e-16, 4.1980e-15,\n 1.0433e-16, 1.7222e-14, -8.0576e-15, 8.6618e-16, 1.8700e-14,\n 1.6775e-13, -1.9811e-15, 5.0264e-14, 5.8134e-16, 1.3031e-13,\n -2.2195e-13, -9.3770e-15, 7.6395e-15, 2.6568e-13, 4.4670e-15,\n 6.2817e-14, -7.0178e-15, 6.5471e-14, 6.3999e-14, 3.4241e-14,\n -9.9902e-15, 2.8176e-14, 2.4701e-13, -5.8406e-15, 7.6571e-14,\n 4.1621e-14, 4.3243e-14, 1.2147e-13, 1.9547e-14, 1.0430e-14,\n 1.4299e-13, 4.6538e-14, -1.9357e-15, 1.1281e-13, 1.3747e-13,\n 9.0455e-14, 1.7024e-13, -2.1575e-15, 2.3049e-13, 6.3953e-14,\n -7.3382e-17, 7.2847e-14, -4.0606e-14, 1.6390e-13, -1.5986e-14,\n -2.7708e-14, 4.9454e-14, -4.5280e-16, -3.7844e-15, 5.8175e-14,\n 6.2932e-14, 1.7689e-13, 8.5233e-14, 1.1760e-13, 3.4758e-14,\n 4.7816e-14, -4.3156e-14, 2.9708e-13, 7.2972e-14, -1.3472e-14,\n -2.3448e-14, 1.0362e-13, -5.3232e-15, 8.3748e-14, 8.1475e-14,\n -7.0022e-16, 4.4652e-14, -9.3703e-15, 6.9952e-14, 8.2722e-14,\n -1.7150e-14, -5.4576e-15, 1.8846e-15, 7.6481e-16, 1.1030e-13,\n 1.4411e-13, 1.4113e-13, 3.6591e-14, 3.9293e-14, -1.5477e-15,\n 3.4426e-14, -6.9176e-15, -1.2363e-14, 8.2177e-15, 1.8432e-14,\n -2.8880e-14, 2.3192e-13, 6.2527e-14, -1.3369e-14, 4.1122e-14,\n 2.8186e-14, -1.6566e-14, -2.0788e-15, -1.4991e-13, 3.7600e-14,\n 1.4980e-13, 2.5825e-15, 1.0065e-14, -1.1679e-14, 8.0167e-16,\n 4.7052e-15, 5.2272e-14, -5.8598e-14, -5.9557e-15, 1.4490e-13,\n -4.0374e-14, 1.2776e-14, -3.0500e-14, 4.4460e-14, -1.4705e-16,\n 2.5777e-14, -1.7819e-16, 1.2338e-14, -1.3386e-14, 6.0501e-14,\n -7.4430e-15, 3.5074e-13, 8.5365e-14, 3.4728e-13, 6.9444e-14,\n 1.7169e-14, 5.6982e-14, -1.5567e-14, -4.0086e-14, 2.8595e-13,\n 3.9454e-15, 1.7703e-13, 4.6030e-15, 6.5699e-14, 1.0786e-13,\n 2.1078e-14, 3.0079e-14, -2.4072e-14, 3.2207e-14, 2.2804e-13,\n -1.9058e-15, 1.8504e-13, 2.3328e-14, 1.8663e-15, 1.0946e-13,\n 6.8585e-16, -1.7402e-15, 8.8241e-14, 1.0545e-13, 9.3312e-14,\n 5.4271e-14, 8.8160e-16, 3.8986e-14, 2.1994e-14, -2.0996e-14,\n -3.6514e-14, 2.0747e-13, 7.4765e-15, -3.5575e-15, 4.5640e-14,\n 2.4622e-15, 1.3286e-13, -8.9308e-16, 1.4903e-13, 1.2112e-13,\n 2.5719e-14, 1.5231e-13, -1.8882e-15, 5.6254e-14, 2.9045e-14,\n 3.5641e-15, 1.3860e-13, 5.0402e-15, -1.6753e-14, 9.7579e-14,\n 8.5979e-14, 6.6059e-15, 2.5551e-14, -1.7683e-15, -2.0643e-15,\n 2.6068e-14, 1.3801e-13, 1.2556e-13, 2.4719e-14, 7.0497e-15,\n 1.5385e-13, 4.2627e-14, 7.8491e-14, 1.3526e-14, 2.6925e-14,\n 1.9324e-18, 1.5331e-13, 1.1718e-13, 3.9810e-14, 1.9388e-13,\n -5.9143e-15, 1.3400e-14, 8.1463e-14, -1.4740e-14, 1.2180e-13,\n -9.7196e-16, 1.2627e-13, -6.6668e-14, -4.4896e-15, 9.3593e-15,\n 2.9299e-14, 4.4717e-14, 3.9613e-14, -3.1771e-15, 3.8014e-14,\n 5.3453e-14, 2.2051e-15, 9.2177e-14, 2.2773e-14, 2.3009e-13,\n 1.6534e-14, 2.9869e-14, 1.6569e-14, 3.1354e-14, 5.6507e-14,\n 4.4177e-14, 2.6577e-13, -6.1121e-15, 1.2920e-15, 1.1498e-14,\n 8.1167e-14, 3.6861e-14, 5.2289e-14, 1.5455e-14, -1.0231e-15,\n -1.6878e-15, 2.3596e-14, 5.7893e-14, -6.3503e-15, -4.6236e-15,\n 1.1921e-13, 1.9081e-13, 2.6002e-14, -1.2860e-14, 3.8587e-16,\n 1.1108e-13, -4.6347e-15, 9.0645e-15, 9.2954e-14, 1.0299e-14,\n 1.4419e-13, 6.0172e-14, 6.8764e-14, 1.6324e-13, -4.9126e-16,\n 3.7182e-14, -1.9004e-15, 5.5518e-14, 1.9744e-15, 3.3740e-14,\n 8.5891e-14, -2.5709e-16, -6.6294e-15, 9.1488e-14, 3.8339e-15,\n 1.5553e-14, -7.2777e-15, 8.2419e-14, -7.7210e-17, 4.5943e-14,\n 8.9172e-15, 1.0449e-13, 9.7976e-14, 2.1809e-13, 4.8281e-14,\n 1.2839e-13, 1.0206e-13, 5.1146e-14, -1.8629e-14, 4.7593e-14,\n -3.6882e-14, 3.9899e-15, 1.4410e-13, 1.7439e-14, -3.6508e-16,\n 1.0295e-14, 2.1663e-13, 5.8912e-15, 9.2976e-14, 5.1757e-15,\n 1.0381e-14, -2.1495e-14, 1.7744e-15, 6.2843e-14, -1.0440e-14,\n 8.5707e-14, -5.5045e-15, 3.0051e-13, -4.6719e-14, 1.4208e-15,\n 5.4743e-14, 9.8476e-14, 9.5095e-15, 1.8075e-13, 1.5503e-13,\n 4.2327e-13, 1.5665e-13, 2.9853e-15, 5.5443e-14, 1.2214e-13,\n 4.9667e-14, -8.3890e-17, -5.1105e-15, -1.4356e-13, 9.5733e-15,\n -1.8752e-15, 5.9438e-15, 9.8630e-14, 4.6109e-14, 1.6028e-14,\n 6.1089e-14, -2.6736e-15, 4.6305e-14, 8.0317e-15, 9.9422e-14,\n 8.1207e-15, 7.4426e-14, 4.4436e-14, -3.4116e-14, 9.1032e-14,\n 2.2754e-14, 1.3271e-13, 7.8994e-15, 3.2427e-14, 9.4743e-14,\n 4.3231e-14, 1.8965e-14, -2.3214e-15, 3.2299e-14, 1.1546e-14,\n 5.2211e-14, -6.0183e-14, 7.0351e-14, 3.8964e-14, 5.8550e-14,\n 2.7685e-14, 1.2354e-13, 6.8071e-15, 2.2054e-13, 1.8124e-13,\n 1.6521e-13, -6.5049e-16, 1.3760e-14, 5.2215e-14, 1.2491e-13,\n 1.9269e-14, 2.9414e-14, 5.7489e-14, 3.4295e-14, 1.7884e-14,\n 1.6283e-15, 6.7401e-14, 7.3799e-14, -1.2596e-14, 6.4156e-14,\n 1.2835e-13, 3.9787e-14, 1.0491e-13, 1.2376e-14, -1.2886e-14,\n 1.1990e-14, -3.1902e-15, -1.7712e-14, 1.6369e-13, 1.4414e-14,\n 4.3623e-13, 3.1027e-14, 1.4489e-14, 1.2810e-13, -1.3468e-14,\n -6.0255e-16, 1.9700e-13, 3.0027e-14, 8.1309e-14, 5.1026e-14,\n 6.2174e-15, 9.1946e-14, 7.7641e-14, 6.5190e-15, -7.4590e-17,\n 1.2341e-14, 2.9121e-16, -5.2298e-15, 6.5144e-14, 9.7343e-14,\n 1.4503e-14, -1.9959e-14, 5.9417e-14, 4.8046e-15, 5.1170e-16,\n 1.1542e-13, 4.2155e-14, 7.3380e-14, 4.4733e-15, 6.9578e-14,\n 7.1621e-14, -2.3872e-14, 7.6038e-15, -8.0524e-17, 9.5908e-14,\n 1.8673e-13, 2.4556e-14, 1.7258e-13, 1.2889e-13, 6.6631e-14,\n 1.2018e-13, 1.0135e-13, 1.1537e-14, -1.2154e-14, -4.5530e-15,\n 7.7197e-15, -5.3481e-14, -2.0455e-15, 1.8455e-14, 5.0725e-14,\n 1.2056e-13, -1.3937e-15, 4.5786e-14, 1.6373e-13, -4.4623e-17,\n 2.7304e-15, 2.8291e-14, 1.3921e-14, 7.7130e-14, 2.0209e-13,\n 4.1033e-14, 5.3736e-14, 6.1645e-14, 1.4284e-13, -4.9897e-15,\n -8.9557e-14, 1.9004e-13, -1.5303e-14, 1.0503e-13, 2.0400e-13,\n 2.1001e-13, 8.3325e-14, -9.3936e-14, 1.8953e-13, 5.7763e-14,\n 2.1700e-14, -1.1222e-14, -3.5108e-15, -1.2382e-14, 1.5466e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.0578e-15, 2.0519e-15, 4.9692e-16, 1.0603e-14, 1.4800e-16, 7.4941e-16,\n 8.0176e-16, 5.4861e-16, 6.4914e-15, 3.1889e-17, 3.3612e-15, 1.1199e-15,\n 1.0629e-15, 9.3046e-16, 4.5028e-16, 2.4104e-15, 2.8953e-15, 5.4530e-17,\n 5.6771e-16, 2.4157e-15, 7.7470e-16, 1.0369e-15, 8.8295e-16, 4.7800e-16,\n 1.3206e-15, 3.4105e-15, 5.7498e-15, 1.6382e-15, 3.9387e-15, 3.0831e-17,\n 2.4370e-16, 2.9518e-15, 7.5148e-17, 5.1206e-16, 1.2168e-17, 2.9480e-15,\n 6.9554e-16, 2.3824e-15, 5.5616e-16, 1.1286e-16, 7.5305e-16, 3.1354e-15,\n 2.1962e-15, 1.6193e-15, 1.6727e-15, 2.1543e-16, 3.5815e-15, 5.3852e-15,\n 1.8367e-15, 1.2802e-15, 2.0758e-15, 3.6131e-15, 2.3043e-17, 2.0380e-16,\n 4.1467e-15, 2.1680e-15, 3.6117e-16, 6.6137e-16, 9.0946e-17, 1.8896e-15,\n 2.8953e-15, 3.0383e-15, 7.7479e-18, 8.2986e-17, 4.9031e-17, 2.6616e-17,\n 2.5063e-15, 3.7941e-15, 1.0394e-16, 7.8697e-18, 1.0015e-15, 3.8406e-16,\n 2.4135e-15, 2.2667e-15, 9.6170e-16, 1.6611e-15, 3.9567e-16, 7.7437e-16,\n 8.6944e-15, 4.1742e-16, 3.1608e-17, 1.4759e-15, 5.9441e-17, 5.9864e-16,\n 1.2155e-15, 1.1585e-15, 5.9587e-16, 2.6810e-15, 1.2289e-15, 3.5156e-15,\n 2.2159e-15, 1.6926e-15, 4.6192e-15, 8.6930e-16, 2.7569e-16, 2.0811e-16,\n 2.8420e-15, 3.1193e-15, 7.7908e-16, 4.2291e-16, 2.1474e-16, 3.4482e-15,\n 1.8584e-16, 1.2033e-15, 3.6576e-17, 3.2325e-16, 3.4229e-15, 5.4287e-16,\n 7.3582e-16, 4.3338e-17, 6.7166e-16, 2.4694e-17, 1.7963e-16, 2.0823e-15,\n 1.6005e-15, 3.1404e-15, 1.0888e-15, 1.5977e-15, 8.7402e-16, 5.3094e-15,\n 2.5864e-15, 3.2178e-15, 1.1114e-15, 3.4177e-17, 1.3103e-16, 2.5934e-17,\n 1.6156e-15, 2.6408e-15, 1.1549e-15, 5.8968e-17, 9.2552e-18, 1.7196e-15,\n 2.6289e-15, 2.0397e-17, 6.5012e-16, 5.0720e-17, 3.1181e-15, 2.2376e-17,\n 2.3452e-17, 9.9231e-16, 4.0683e-17, 3.4784e-15, 3.4632e-15, 1.3716e-17,\n 3.9459e-15, 2.7840e-15, 7.1532e-16, 7.0303e-16, 9.8183e-16, 9.8633e-16,\n 2.2165e-15, 1.0351e-17, 3.8648e-15, 2.1337e-15, 1.3276e-15, 2.5978e-17,\n 6.8590e-16, 8.2673e-17, 1.0685e-15, 7.9125e-16, 6.7768e-15, 2.1621e-16,\n 1.2473e-15, 3.1257e-16, 2.3313e-15, 1.8116e-15, 3.0332e-16, 6.8262e-17,\n 1.1327e-15, 3.7324e-15, 5.3724e-15, 7.3496e-16, 2.0934e-15, 3.8113e-15,\n 4.9944e-17, 3.4128e-15, 1.5568e-15, 8.2399e-16, 2.7500e-15, 8.1668e-16,\n 7.1609e-16, 7.7787e-16, 3.7579e-15, 8.5953e-16, 6.6516e-15, 4.2550e-15,\n 1.5649e-15, 2.5723e-16, 1.8969e-15, 2.2850e-15, 2.4995e-15, 5.5850e-15,\n 1.7560e-16, 1.7110e-17, 1.6237e-15, 2.6146e-15, 5.6084e-17, 3.8297e-17,\n 6.8987e-17, 1.6304e-15, 2.0049e-15, 1.6013e-16, 9.0368e-16, 2.9125e-17,\n 1.3211e-15, 6.1304e-15, 1.6571e-17, 3.0816e-15, 4.6845e-15, 2.0823e-15,\n 1.4423e-16, 1.2684e-17, 5.7313e-16, 4.3977e-15, 8.7789e-16, 4.3877e-17,\n 1.1905e-15, 7.3953e-16, 5.1175e-18, 1.8952e-15, 1.2972e-15, 1.7885e-15,\n 2.6912e-15, 3.7375e-15, 8.9705e-16, 4.0827e-15, 2.2519e-15, 3.7610e-17,\n 3.0555e-16, 4.6854e-15, 4.0821e-17, 6.4169e-15, 5.6417e-16, 3.1755e-15,\n 2.9963e-15, 2.0210e-16, 3.7600e-15, 1.0848e-15, 4.5864e-15, 1.9389e-17,\n 1.8000e-17, 1.0038e-15, 9.0493e-17, 2.2778e-17, 8.7433e-16, 1.4217e-15,\n 9.7695e-16, 1.7759e-15, 5.3216e-15, 3.6064e-16, 1.0248e-15, 4.4068e-16,\n 2.7783e-15, 3.4031e-15, 2.2533e-16, 3.6354e-16, 2.6454e-16, 7.0706e-17,\n 9.2983e-16, 1.8498e-15, 1.6971e-15, 4.3168e-16, 7.6787e-18, 4.6772e-15,\n 1.4518e-16, 1.7649e-15, 7.2212e-16, 1.6006e-15, 1.0502e-16, 1.5399e-15,\n 2.4345e-17, 9.6804e-16, 1.4613e-16, 2.4202e-15, 1.1054e-17, 3.7786e-15,\n 2.2187e-15, 1.3397e-17, 7.8264e-16, 1.7848e-15, 1.3677e-15, 1.4701e-15,\n 3.6021e-15, 6.4722e-16, 1.1001e-14, 5.2777e-16, 3.9681e-16, 4.7341e-17,\n 4.3723e-15, 8.1329e-17, 7.7557e-16, 1.7030e-15, 4.2715e-16, 5.1784e-17,\n 2.8663e-17, 1.6019e-17, 1.5342e-15, 1.7919e-16, 5.1219e-16, 2.0395e-15,\n 2.3325e-15, 5.7459e-16, 3.8591e-16, 3.5253e-15, 5.0565e-18, 7.8686e-16,\n 1.1368e-16, 6.1314e-17, 4.0965e-17, 8.1721e-17, 1.0116e-15, 1.0062e-15,\n 1.6574e-16, 9.9143e-15, 5.1872e-15, 7.3444e-15, 5.9033e-16, 2.6087e-17,\n 2.6363e-15, 3.2780e-15, 2.5928e-16, 1.5990e-17, 1.7018e-15, 1.4639e-15,\n 4.5760e-16, 1.2792e-15, 9.2068e-17, 5.3028e-17, 2.8917e-15, 3.7364e-15,\n 1.2920e-15, 4.1794e-15, 9.9735e-16, 3.9023e-16, 4.1276e-15, 1.4291e-16,\n 2.4493e-15, 7.2039e-16, 3.5147e-16, 2.4578e-15, 3.0336e-16, 1.0181e-15,\n 2.5193e-15, 3.7994e-16, 2.8305e-15, 1.1235e-15, 7.1730e-15, 1.9404e-16,\n 3.0701e-17, 4.8318e-15, 5.9504e-16, 5.2737e-15, 1.7229e-17, 4.7314e-15,\n 4.5719e-15, 1.1125e-15, 9.8774e-16, 1.6315e-15, 4.6307e-15, 9.7347e-18,\n 3.4937e-15, 1.4524e-15, 2.2730e-15, 2.3655e-16, 1.0803e-15, 6.0632e-16,\n 4.2074e-16, 2.4298e-15, 4.8353e-15, 5.4114e-16, 4.0274e-15, 8.9863e-16,\n 5.3778e-16, 2.4347e-15, 4.2095e-16, 2.0369e-15, 2.5096e-15, 3.1960e-15,\n 9.4919e-16, 3.6523e-15, 8.3378e-16, 4.7942e-17, 1.2042e-15, 5.9699e-16,\n 4.4194e-16, 3.0861e-15, 2.2401e-17, 2.5681e-15, 3.3265e-17, 2.1017e-15,\n 6.1726e-16, 2.9409e-17, 3.7285e-16, 1.5308e-16, 4.9683e-17, 3.9747e-16,\n 3.7414e-15, 7.0459e-15, 1.4383e-17, 1.4634e-16, 4.8754e-15, 3.3034e-16,\n 6.0502e-15, 8.8329e-15, 6.0538e-15, 5.0311e-16, 4.7512e-16, 1.6609e-15,\n 2.1151e-15, 6.7404e-16, 7.6961e-16, 2.9439e-15, 6.8246e-18, 3.6031e-15,\n 7.5693e-16, 8.6234e-16, 6.4729e-17, 1.9594e-15, 1.4539e-15, 3.5192e-16,\n 3.1975e-16, 3.4116e-16, 2.8256e-15, 1.7218e-16, 4.2394e-16, 1.8869e-15,\n 4.3121e-15, 8.1481e-16, 3.1814e-17, 5.6110e-18, 3.2527e-15, 1.0449e-17,\n 2.8806e-16, 3.6523e-15, 4.3058e-16, 1.4705e-15, 5.0289e-15, 2.5963e-15,\n 1.0052e-15, 1.1961e-16, 8.3622e-16, 2.5497e-16, 6.9009e-15, 4.2994e-17,\n 1.2437e-15, 3.6749e-16, 7.8099e-17, 7.7842e-17, 6.8676e-15, 7.1267e-15,\n 1.2641e-16, 6.7799e-18, 7.0747e-15, 8.0911e-18, 3.5651e-15, 2.1101e-16,\n 3.6979e-15, 5.5967e-15, 3.7205e-15, 1.1802e-15, 9.7404e-15, 4.9926e-15,\n 6.5350e-16, 2.5308e-16, 3.3432e-17, 3.7202e-15, 2.4153e-17, 2.5458e-15,\n 5.3067e-16, 1.1307e-17, 6.1155e-16, 2.8964e-16, 6.1313e-16, 1.9156e-15,\n 3.1853e-16, 5.6603e-17, 9.0640e-16, 1.4023e-15, 9.3027e-16, 2.0096e-16,\n 2.3325e-16, 4.0361e-15, 9.2886e-16, 1.2959e-15, 8.3418e-16, 3.1306e-16,\n 2.3594e-15, 6.9645e-16, 1.3552e-15, 1.1242e-16, 1.0672e-14, 3.9627e-15,\n 1.4871e-16, 1.0913e-15, 8.0256e-16, 2.8515e-16, 1.3168e-17, 6.2539e-16,\n 1.8591e-15, 3.0896e-16, 1.6531e-15, 1.2967e-16, 1.5661e-15, 3.7671e-15,\n 5.9384e-15, 2.3062e-15, 1.3003e-16, 1.1203e-15, 1.5750e-17, 1.3481e-16,\n 2.9224e-16, 1.2741e-15, 7.2302e-17, 1.5029e-15, 6.4196e-15, 8.0914e-15,\n 9.4944e-16, 4.4239e-17, 1.7481e-16, 2.6417e-15, 8.0736e-16, 1.6497e-16,\n 3.8111e-17, 7.7847e-16, 3.9427e-16, 3.7248e-15, 1.6410e-16, 3.0040e-15,\n 2.0354e-15, 2.5541e-15, 2.0189e-16, 2.9931e-15, 4.6757e-15, 5.7104e-17,\n 2.9540e-15, 5.0528e-16, 8.5354e-18, 1.5140e-16, 2.0497e-15, 1.9567e-15,\n 1.3015e-15, 1.9703e-16, 2.6903e-16, 1.6469e-16, 2.4549e-16, 1.1412e-15,\n 2.5165e-15, 1.1537e-15, 1.2181e-17, 9.4024e-16, 1.3745e-15, 8.1476e-16,\n 2.5470e-17, 1.2571e-15, 2.7458e-15, 1.8859e-16, 2.7662e-17, 3.1412e-17,\n 5.1783e-15, 2.6909e-16, 1.1329e-14, 2.4073e-15, 3.2021e-17, 6.3268e-15,\n 1.4933e-16, 3.1800e-17, 3.9754e-15, 1.2088e-15, 2.7504e-16, 7.5270e-16,\n 4.2389e-16, 4.8391e-16, 8.3578e-15, 5.7327e-16, 2.3922e-15, 1.0434e-15,\n 1.7391e-15, 5.4464e-16, 6.5557e-16, 2.1443e-15, 4.6299e-16, 1.1006e-15,\n 3.8545e-15, 1.1799e-16, 1.2355e-15, 3.0368e-15, 4.2552e-16, 3.6203e-15,\n 2.2196e-15, 1.1927e-15, 4.1837e-16, 9.2620e-16, 7.7727e-16, 1.3242e-17,\n 4.7555e-15, 3.6863e-15, 3.7821e-15, 3.6106e-15, 5.0598e-16, 2.3074e-15,\n 2.4092e-15, 1.9108e-15, 5.8628e-16, 7.7664e-16, 4.0590e-16, 1.7160e-17,\n 1.9573e-15, 1.0553e-17, 1.4576e-15, 2.0400e-15, 1.9906e-15, 5.6379e-15,\n 2.3006e-15, 4.3018e-17, 2.7983e-16, 7.6723e-16, 3.6654e-16, 9.4799e-16,\n 5.6258e-15, 9.7613e-16, 2.0963e-15, 6.5059e-17, 1.2484e-15, 2.5753e-15,\n 1.7266e-16, 2.0867e-15, 7.5582e-17, 9.5541e-16, 3.4032e-17, 1.7858e-15,\n 5.3491e-15, 1.7581e-17, 7.5321e-16, 2.5973e-15, 7.4707e-17, 1.1808e-16,\n 2.3451e-17, 3.1823e-15, 2.3912e-15, 7.0435e-15], device='cuda:0')" + }, + "42": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -3.3950e-17, 2.6673e-17, ..., 1.7337e-15,\n -2.3096e-16, 3.9109e-17],\n [ 5.6052e-45, -7.0861e-17, 2.6871e-17, ..., 8.9128e-16,\n 1.7711e-15, 2.4714e-17],\n [-5.6052e-45, -5.8161e-16, 6.4387e-17, ..., 1.1596e-15,\n 3.1143e-15, 3.4890e-17],\n ...,\n [ 5.6052e-45, -1.1035e-16, -3.5447e-17, ..., -1.6120e-15,\n -8.7908e-16, -8.5971e-18],\n [ 5.6052e-45, 6.4455e-16, -2.8265e-17, ..., -7.7980e-16,\n -2.1986e-15, -3.0300e-17],\n [ 5.6052e-45, -3.1595e-17, 2.3586e-18, ..., 3.0740e-17,\n 6.2428e-17, -3.6980e-19]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.6304e-16, 1.4313e-15, 1.6168e-15, ..., 2.2654e-17, 1.9072e-16,\n 8.1465e-16],\n [2.8831e-15, 4.4847e-15, 5.3185e-15, ..., 1.0087e-17, 1.9294e-16,\n 5.1810e-15],\n [3.4596e-16, 1.1227e-15, 2.3307e-15, ..., 2.8328e-18, 3.3324e-17,\n 7.3625e-16],\n ...,\n [5.8005e-17, 7.0866e-17, 5.7783e-17, ..., 1.7700e-17, 1.2133e-17,\n 2.6919e-17],\n [8.6816e-16, 2.9329e-15, 2.4550e-15, ..., 2.7048e-17, 3.3047e-16,\n 1.1427e-15],\n [9.7882e-18, 4.0538e-17, 2.8158e-17, ..., 1.0822e-18, 5.5857e-18,\n 1.2839e-17]], device='cuda:0')" + }, + "43": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-3.8774e-14, 5.0736e-13, 4.6814e-12, 3.9989e-12, -6.2240e-13,\n 2.4723e-13, -3.2865e-12, -7.3053e-13, -5.9784e-13, -4.5054e-13,\n 2.9121e-12, -1.3963e-12, 7.1005e-13, -1.4062e-12, 3.9183e-12,\n 2.8180e-12, 1.2236e-12, -5.7250e-13, 1.4039e-12, -9.3803e-13,\n -7.9046e-13, 1.6985e-12, 8.1445e-13, -1.2446e-12, 1.3461e-12,\n 6.4520e-13, -4.5909e-13, -3.8032e-13, -5.4913e-13, 9.8046e-13,\n -4.6588e-13, 1.7141e-12, -6.4064e-13, -3.0112e-13, 1.0490e-12,\n -1.4396e-12, -2.9995e-13, 3.5918e-12, 3.0264e-12, 6.1007e-13,\n 8.0702e-13, -5.7378e-13, -8.0567e-13, -2.0735e-12, -6.6098e-13,\n -1.4257e-12, 8.1464e-14, 1.9659e-12, -1.4893e-13, -3.3077e-13,\n -7.9671e-13, -3.3533e-13, 1.1026e-12, 1.1663e-13, 2.1000e-12,\n 1.0030e-12, 1.1780e-12, -1.0018e-12, 6.6708e-13, 2.3608e-12,\n -1.4741e-12, -3.5747e-13, 1.8249e-12, 1.7255e-12, -5.6547e-13,\n 6.7332e-13, -9.7699e-13, -8.5290e-13, -9.8246e-13, 2.7924e-12,\n -9.8926e-13, -1.0248e-12, -6.1393e-13, 5.1653e-13, 3.9545e-13,\n -2.0672e-12, -5.9916e-13, -5.7051e-13, 1.1822e-12, 1.0726e-12,\n -4.7837e-13, -7.1839e-13, 7.4447e-12, 1.9195e-12, 8.8140e-13,\n 9.2487e-13, 1.3760e-12, -3.5374e-13, -3.2989e-12, -4.4718e-13,\n -7.6126e-13, 1.5479e-12, -3.9187e-12, -5.2948e-13, 6.5205e-13,\n 1.0151e-13, -5.5285e-13, -6.8215e-13, -7.1613e-13, -3.2147e-13,\n -3.3897e-12, -2.3493e-12, 6.3452e-13, 2.2257e-12, 3.3911e-13,\n -1.8746e-13, -1.2189e-12, 4.9056e-13, -1.2608e-12, -6.1293e-13,\n -2.3107e-12, 1.0122e-12, -4.8799e-14, 1.5824e-13, -2.5084e-13,\n -4.0228e-12, -5.3791e-13, 8.0572e-13, -9.2642e-13, 1.8253e-12,\n 8.5838e-13, 8.1493e-13, 3.1048e-12, 2.1735e-12, 5.5811e-12,\n -4.2360e-12, -6.1282e-13, -4.4281e-14, -2.1664e-13, 4.5884e-13,\n -2.4185e-13, 2.8270e-13, 6.8734e-13, -8.1076e-13, 5.7289e-13,\n 4.5398e-13, 7.6131e-13, -2.6759e-13, 6.2439e-12, 1.1304e-12,\n -8.4785e-14, -7.5019e-13, -6.3277e-12, -3.8122e-13, 3.3253e-13,\n 1.0874e-12, -6.9941e-13, -5.6786e-13, 2.8491e-13, -4.3146e-13,\n 2.1158e-13, 7.8474e-13, 2.7585e-14, 4.0973e-12, 4.0400e-12,\n 4.7361e-14, -4.4498e-13, -5.7777e-13, -3.4619e-13, 2.0650e-14,\n -5.1301e-13, -6.3986e-13, 8.0422e-13, 1.1183e-12, 2.1162e-13,\n 3.8598e-12, -5.3742e-13, 5.4091e-13, -1.9564e-13, -8.8501e-13,\n -7.1339e-13, 3.7622e-12, -4.5071e-12, 1.2416e-12, 2.0185e-12,\n -1.8345e-13, 4.2833e-13, -1.9722e-12, -8.4062e-13, -5.7439e-13,\n 7.5633e-13, 8.5470e-13, -6.2158e-13, 1.0430e-13, -4.6713e-13,\n -6.5998e-13, -6.4613e-13, -2.4175e-12, -8.2831e-13, -1.3886e-12,\n 1.2960e-12, 1.2559e-12, -8.9256e-14, -6.3824e-13, -1.7353e-12,\n -1.3885e-12, -5.1964e-13, 1.0290e-12, -4.1277e-13, -4.2490e-13,\n -1.0931e-13, -1.0125e-12, -9.0683e-13, -2.6789e-12, 7.6663e-13,\n -6.1542e-12, 1.5609e-12, -1.9326e-12, 4.6981e-12, -1.0095e-12,\n -1.7756e-13, -5.0343e-13, -1.2854e-13, 1.9386e-13, 8.0594e-13,\n 1.0901e-12, 1.5888e-12, 3.1483e-12, -7.1529e-13, -3.6145e-13,\n -1.3714e-12, -7.1525e-13, 8.8348e-14, -1.1993e-12, -1.1638e-12,\n 1.3479e-12, -6.0195e-13, -1.5531e-12, 2.5383e-12, 7.9865e-13,\n 2.9203e-12, 1.6671e-12, -6.1314e-13, 8.8467e-13, -8.0693e-13,\n -2.0133e-12, -7.0362e-13, -3.2476e-12, -3.0224e-13, 1.0878e-12,\n 3.1180e-12, -4.5527e-13, -3.6061e-13, -2.0028e-12, 8.2291e-13,\n 6.8262e-13, 1.4850e-12, -3.9957e-13, -3.2179e-13, -9.1623e-13,\n -2.8796e-13, -4.3523e-12, -7.3991e-13, 1.0340e-11, 2.2297e-12,\n 1.3528e-12, 4.2709e-12, 1.2473e-13, -5.4704e-13, 8.1040e-13,\n 1.9573e-13, 3.7338e-13, 1.3304e-12, -1.6488e-13, -8.3133e-13,\n 1.1704e-13, -1.5340e-12, 2.8496e-12, 3.2107e-12, 2.9715e-12,\n -9.0709e-13, 7.4612e-13, -9.4205e-13, -6.3868e-13, -7.2629e-13,\n -6.3377e-13, -1.2726e-12, -1.5561e-12, -2.1113e-12, -5.5146e-13,\n -1.2000e-12, -5.3775e-13, 1.3761e-12, -2.0046e-12, -1.2804e-13,\n -7.3420e-13, 8.3988e-13, -3.3152e-12, 1.9725e-12, 1.1450e-12,\n -1.6762e-12, -2.4643e-12, -8.9496e-13, 8.8253e-13, -3.8837e-12,\n -5.2502e-12, 6.1575e-13, -2.7910e-12, -4.5927e-12, -8.4082e-13,\n -4.7855e-12, -1.4729e-12, -2.0437e-12, -9.5267e-13, -9.1104e-13,\n -2.7181e-13, -2.8793e-12, -5.9169e-12, 1.5798e-12, 2.1247e-12,\n -2.2160e-12, 3.6241e-12, -6.4113e-13, 2.8414e-12, -8.9518e-13,\n -6.8072e-13, -8.4726e-13, 4.1053e-12, -4.8437e-12, 5.9752e-12,\n -1.8513e-13, 2.3844e-12, -9.3640e-13, 4.3783e-13, 1.4823e-12,\n -7.5839e-13, -1.1133e-13, -6.7684e-13, -5.7904e-13, 6.5330e-12,\n 2.4198e-12, 1.1699e-12, -1.0600e-12, -1.0078e-12, -3.0386e-12,\n -2.9718e-12, 3.7781e-14, -1.0491e-12, 2.8082e-12, -3.5244e-13,\n -4.8225e-13, 7.1383e-13, -2.3012e-12, -6.3498e-14, -1.0736e-12,\n -1.1372e-13, -1.3559e-12, 1.2284e-12, -7.8133e-13, 5.8223e-13,\n -6.3521e-13, -6.4515e-13, 7.2203e-13, -5.4749e-13, -8.7173e-14,\n 1.1007e-12, 1.9044e-12, -3.1434e-12, 2.1330e-12, 3.9859e-13,\n -1.1854e-13, 1.5570e-12, -7.2160e-13, 1.1796e-12, 5.1301e-12,\n 5.5982e-13, -4.9070e-13, -8.5168e-13, -1.4042e-12, 5.3892e-13,\n -5.5505e-13, -4.7770e-13, -2.8043e-13, 2.9253e-12, -1.6408e-12,\n 4.1003e-12, -5.6055e-13, -1.8697e-13, -2.1981e-15, -3.8778e-13,\n -1.7435e-12, -1.8338e-13, 2.6278e-12, -1.0759e-12, -1.8863e-13,\n -3.3182e-12, 8.0672e-13, 2.0554e-12, -4.5539e-13, 6.8927e-12,\n 4.5188e-13, 1.6716e-12, -4.0465e-12, -6.8097e-13, -4.3944e-13,\n 7.6759e-12, 1.3592e-13, -2.3314e-13, -6.6470e-13, -8.2265e-13,\n 2.6677e-12, -5.5677e-14, 9.7830e-13, -1.1475e-12, -2.4214e-13,\n 2.3366e-13, -9.0351e-13, -1.7846e-12, -2.2215e-12, 3.5863e-12,\n -7.5420e-13, 2.3508e-12, -2.4269e-12, -1.2946e-13, -6.6471e-13,\n -2.4863e-12, -3.0071e-13, 2.5052e-12, 3.1899e-12, -5.4325e-13,\n -1.0661e-13, 2.2559e-13, -6.0745e-13, 1.2551e-13, 7.4442e-13,\n -2.0320e-13, -5.8069e-13, -5.3744e-13, 1.1268e-13, 1.5751e-12,\n -1.0969e-13, 8.7771e-14, 7.8487e-13, -1.3024e-14, -2.8253e-14,\n 5.7708e-13, -2.8690e-13, -8.6807e-13, -1.9822e-14, -3.0070e-12,\n -6.3614e-13, 3.1490e-13, -9.5537e-13, -2.1124e-12, 2.8683e-12,\n -7.8902e-13, -5.4051e-12, -6.3593e-13, 1.5192e-12, -1.0650e-13,\n -6.4131e-13, -1.4101e-12, 5.1065e-14, -6.2428e-14, -1.2172e-12,\n -3.3571e-13, 1.6584e-12, 1.2066e-12, -5.6725e-13, -1.0346e-12,\n 2.3114e-12, 1.1778e-12, -2.8940e-12, -1.4627e-13, -5.0600e-13,\n 1.8672e-12, -1.6747e-12, -2.7665e-13, -6.3920e-13, -3.0301e-12,\n -4.5479e-13, 1.8837e-12, -2.2490e-12, -1.2095e-12, 6.7761e-13,\n -8.4418e-13, 1.5726e-13, -8.0324e-13, -9.5137e-13, -7.9990e-13,\n 1.7766e-12, -4.4041e-12, 3.4644e-12, -3.6989e-12, -1.0852e-12,\n 1.8278e-13, 5.2669e-12, 1.0994e-12, 5.1690e-12, -6.2405e-13,\n 7.0203e-12, 4.5995e-12, -7.4847e-13, 2.0941e-12, 1.9124e-12,\n 5.0407e-13, -1.8242e-12, -3.9685e-13, -6.1075e-12, -5.9765e-13,\n 1.3581e-12, 4.1248e-13, 1.9213e-12, -3.0177e-13, -1.1129e-12,\n -6.1629e-13, -7.5700e-13, -5.2656e-13, -8.2380e-13, -2.2884e-13,\n 6.2476e-13, -3.9064e-13, -2.1985e-13, -3.0728e-12, -2.5578e-12,\n -2.9285e-13, -1.7133e-13, -4.4044e-12, 5.2166e-13, 9.3516e-13,\n 1.3706e-12, -6.2379e-13, -2.4187e-12, 2.7340e-14, -4.9026e-13,\n -1.5236e-12, -1.5040e-12, -4.8506e-13, -7.1722e-13, -1.0589e-12,\n -8.2148e-13, 4.8295e-12, -1.8184e-12, 5.2288e-12, -5.2874e-13,\n 5.7007e-13, 4.7557e-13, 1.9979e-12, 6.1465e-13, 6.0275e-14,\n -4.4375e-13, -1.0476e-12, -4.5093e-13, -4.6352e-15, -1.7252e-12,\n -4.6711e-13, 3.0735e-12, -6.1955e-13, 1.3279e-12, -6.2096e-13,\n 3.2983e-12, -6.7306e-13, -8.4829e-13, -1.3921e-12, -2.3001e-12,\n -5.1339e-13, -1.1095e-12, -1.3082e-12, -4.1571e-13, 3.2413e-12,\n 1.4377e-12, -3.1180e-13, 7.8528e-13, -5.9876e-13, -1.9339e-13,\n -1.2116e-12, -4.5997e-13, -3.7582e-13, 7.5784e-13, 1.5917e-12,\n -1.0973e-13, 2.1935e-13, -6.4912e-14, -3.8683e-13, -4.3765e-13,\n -5.3718e-13, -4.1748e-13, -2.1728e-14, 2.0748e-12, 4.0374e-12,\n -9.0020e-13, -1.8739e-12, 1.9710e-12, 1.2933e-12, 7.6726e-13,\n 5.5520e-13, -1.6013e-12, 2.7455e-12, -2.4724e-12, -3.5911e-13,\n -7.3959e-13, -4.8095e-13, 1.0048e-12, -5.5125e-13, 9.4091e-13,\n -7.5263e-13, 1.2533e-12, 3.0372e-12, 2.8423e-13, 1.2901e-12,\n -9.0071e-13, 2.0557e-12, -5.3033e-13, 3.5728e-13, -2.2209e-12,\n -8.0667e-13, -3.6368e-12, -5.4906e-13, -1.6794e-12, -4.9345e-14,\n -4.2675e-13, -4.4309e-12, -1.9331e-12, -5.6990e-13, -1.0072e-12,\n -5.9290e-13, -4.9802e-13, -2.6977e-12, -1.6098e-12, 2.7925e-12,\n -2.0003e-12, 4.0889e-13, -5.2382e-13, -1.1184e-12, -8.5764e-13,\n -2.8995e-12, 1.2814e-12, 6.5107e-14, 4.5236e-12, -2.8741e-13,\n 2.4462e-12, 1.3009e-12, -1.0677e-12, 2.1874e-12, -6.3619e-13,\n -6.5501e-13, -5.5885e-13, -3.5659e-13, -5.4352e-12, -1.5794e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.7882e-12, 3.9911e-12, 1.3239e-12, 1.5308e-12, 1.3282e-13, 1.6469e-13,\n 2.5083e-12, 9.3294e-14, 3.1316e-13, 2.3553e-13, 4.2793e-12, 2.0989e-12,\n 3.4035e-12, 1.3533e-12, 1.5967e-12, 3.4656e-12, 2.2777e-12, 1.1402e-12,\n 3.0136e-12, 6.1146e-13, 1.3399e-13, 1.0183e-12, 3.3146e-13, 2.9931e-13,\n 6.4052e-12, 1.1015e-12, 1.2237e-13, 6.4730e-13, 3.3455e-12, 2.0265e-12,\n 6.9662e-14, 2.9926e-12, 3.6397e-12, 3.5607e-12, 5.3316e-13, 1.6388e-12,\n 6.4418e-12, 3.0362e-13, 2.3562e-12, 1.2304e-13, 5.1149e-13, 9.1588e-13,\n 3.0936e-13, 1.4457e-12, 1.5311e-13, 2.9711e-12, 2.6248e-11, 7.0825e-12,\n 1.9919e-12, 3.2984e-12, 1.0945e-13, 3.5727e-13, 3.1492e-13, 6.1618e-13,\n 2.4736e-11, 3.9925e-13, 2.3346e-13, 1.5959e-13, 5.6238e-12, 6.1298e-12,\n 1.5642e-12, 3.4500e-12, 9.7547e-13, 1.6103e-11, 8.2159e-14, 4.2672e-14,\n 2.2311e-12, 2.1612e-12, 8.3531e-13, 1.9987e-12, 2.0169e-12, 1.3609e-13,\n 4.5515e-12, 4.0635e-12, 4.1173e-13, 2.2857e-12, 5.9600e-13, 1.9836e-12,\n 1.2096e-11, 5.4246e-12, 6.5826e-13, 2.3956e-12, 1.5341e-12, 3.5016e-12,\n 1.6016e-13, 5.1686e-12, 2.2755e-12, 7.6149e-13, 7.5520e-13, 1.3964e-12,\n 4.2343e-13, 1.9117e-12, 1.0759e-12, 5.6976e-13, 3.5124e-13, 5.9592e-13,\n 5.5256e-12, 1.0766e-13, 4.1098e-12, 4.2605e-13, 2.6079e-13, 1.4947e-12,\n 6.4240e-12, 5.5825e-13, 8.3288e-12, 1.2133e-12, 1.0508e-13, 1.5138e-12,\n 1.0146e-12, 1.3837e-12, 2.8782e-12, 4.8713e-12, 5.2175e-13, 6.3033e-14,\n 2.1974e-13, 9.2990e-13, 5.9225e-13, 4.2729e-12, 5.2410e-13, 1.4946e-11,\n 1.8951e-12, 9.8833e-14, 3.6388e-13, 1.9482e-12, 2.0575e-13, 1.0688e-11,\n 1.5847e-13, 1.3301e-13, 3.1021e-12, 9.4586e-13, 2.7348e-12, 4.8497e-14,\n 3.2543e-12, 2.9038e-12, 1.0971e-12, 8.2800e-13, 2.1198e-12, 1.4476e-13,\n 3.2106e-12, 3.8928e-12, 5.7444e-13, 1.8747e-12, 5.6646e-12, 5.1565e-13,\n 2.2407e-13, 9.8300e-13, 4.7187e-13, 3.4992e-12, 9.6484e-14, 9.4319e-13,\n 7.6100e-13, 6.3505e-14, 6.3554e-12, 9.8056e-12, 3.1052e-12, 1.2209e-12,\n 3.3532e-12, 1.1784e-13, 1.4572e-13, 1.4223e-12, 8.9610e-13, 1.3189e-12,\n 2.3220e-13, 7.0026e-12, 2.4203e-13, 3.4077e-12, 1.0305e-13, 1.0851e-11,\n 1.0244e-11, 3.4974e-12, 1.7589e-13, 3.1174e-12, 2.4932e-12, 8.8057e-13,\n 2.4545e-11, 2.5598e-12, 6.1806e-12, 1.2357e-12, 2.2627e-12, 1.8264e-13,\n 2.4985e-12, 3.8015e-13, 6.8614e-13, 4.6417e-12, 2.7134e-12, 1.0814e-12,\n 1.9324e-13, 2.8455e-13, 1.7637e-12, 2.7248e-13, 1.4873e-12, 2.3576e-12,\n 3.4346e-12, 1.9562e-13, 3.3834e-12, 2.2818e-12, 1.0619e-13, 4.4758e-12,\n 8.1004e-12, 8.3756e-12, 2.8837e-12, 2.5209e-12, 3.8060e-12, 6.5088e-13,\n 1.4320e-12, 3.7801e-12, 6.1417e-12, 6.4584e-13, 2.5220e-12, 2.2907e-12,\n 8.3519e-13, 5.9820e-13, 1.0568e-12, 6.4910e-14, 5.6710e-13, 4.7906e-12,\n 1.7275e-12, 3.6504e-12, 2.3020e-12, 3.2320e-12, 6.3933e-14, 7.5386e-14,\n 6.9278e-12, 6.7083e-12, 1.0803e-13, 8.9592e-12, 3.1192e-12, 2.0363e-12,\n 5.1819e-14, 1.3926e-11, 3.6843e-12, 3.8578e-12, 2.6538e-12, 4.4633e-12,\n 6.9754e-13, 6.7691e-13, 6.4314e-12, 3.1897e-13, 5.8260e-12, 2.6667e-12,\n 4.4252e-12, 6.3713e-13, 1.3096e-12, 2.6779e-12, 1.8082e-12, 2.9409e-13,\n 1.2618e-11, 4.1593e-13, 1.7812e-12, 1.5632e-13, 6.2599e-13, 3.3129e-12,\n 1.4582e-12, 9.4810e-12, 4.0969e-12, 3.6330e-13, 4.9336e-12, 5.0958e-12,\n 1.6262e-12, 2.9890e-13, 1.8297e-13, 5.8573e-12, 6.9984e-13, 1.7261e-12,\n 1.7736e-13, 1.4125e-13, 2.5596e-13, 2.6945e-12, 2.5658e-12, 1.0351e-13,\n 7.2455e-14, 1.6203e-13, 1.0861e-12, 4.7274e-12, 8.1596e-14, 8.2614e-12,\n 1.2436e-13, 4.3516e-12, 4.0461e-13, 1.9022e-12, 1.2833e-12, 3.7433e-13,\n 8.6747e-13, 1.4621e-12, 3.0885e-13, 2.8492e-13, 3.9425e-13, 5.3431e-12,\n 1.6827e-13, 6.1060e-12, 6.3274e-13, 1.4103e-11, 7.8029e-13, 8.3324e-12,\n 2.3851e-12, 1.0735e-12, 1.5493e-11, 2.5349e-12, 1.9860e-12, 2.4280e-12,\n 5.0845e-13, 1.2372e-12, 3.8799e-12, 1.1804e-12, 2.8579e-12, 7.0098e-12,\n 1.5994e-11, 1.1660e-11, 3.9904e-12, 4.2741e-12, 3.6475e-13, 8.4635e-13,\n 3.1756e-13, 1.1874e-13, 1.8358e-12, 1.5695e-13, 1.3287e-12, 3.0247e-12,\n 5.8909e-13, 1.0848e-11, 3.5643e-12, 6.5706e-13, 2.8135e-12, 4.4060e-13,\n 1.6237e-12, 1.2466e-13, 7.2331e-12, 1.0643e-12, 1.3282e-12, 2.2936e-12,\n 1.4663e-13, 5.8630e-12, 8.3400e-13, 3.3079e-13, 1.5749e-11, 7.0437e-13,\n 5.5464e-12, 4.5981e-13, 8.4557e-13, 7.8521e-13, 2.1027e-13, 1.2157e-12,\n 1.4007e-11, 1.0257e-13, 8.7353e-13, 9.2530e-14, 1.0005e-13, 5.8281e-12,\n 3.8182e-13, 1.2578e-11, 8.7930e-13, 6.9844e-12, 3.3337e-13, 3.8083e-13,\n 1.8115e-11, 3.2120e-12, 1.1451e-12, 3.6186e-12, 4.1957e-14, 1.6015e-13,\n 3.4829e-12, 1.1377e-12, 2.4677e-12, 2.3323e-13, 9.1286e-13, 2.3270e-13,\n 2.0977e-12, 3.7316e-12, 2.5802e-12, 2.9510e-12, 7.2425e-13, 1.6603e-13,\n 4.8726e-13, 4.3040e-12, 4.6687e-12, 1.8964e-11, 7.7193e-12, 5.2165e-12,\n 1.0757e-13, 4.0543e-12, 1.0612e-12, 2.5284e-12, 1.0346e-12, 1.4542e-13,\n 6.1484e-13, 6.6648e-12, 2.8828e-12, 1.7150e-12, 8.0890e-12, 1.9087e-12,\n 1.4838e-12, 7.2193e-12, 1.8257e-12, 6.7981e-13, 1.4905e-12, 2.3617e-13,\n 5.6288e-13, 1.4064e-13, 9.4287e-13, 5.5855e-12, 1.0929e-12, 9.1369e-13,\n 8.7732e-13, 3.6589e-12, 1.8590e-12, 5.9558e-13, 1.2409e-13, 3.4987e-12,\n 3.0666e-13, 1.5638e-12, 1.6505e-12, 3.8887e-12, 7.9586e-13, 2.1139e-13,\n 1.0064e-12, 2.7580e-12, 2.1037e-13, 5.1860e-13, 5.1346e-13, 1.5818e-12,\n 1.7457e-12, 2.2074e-13, 4.6260e-13, 8.0147e-13, 4.6036e-13, 1.3706e-13,\n 6.7673e-13, 4.5194e-12, 2.4421e-12, 1.3875e-13, 9.5533e-12, 1.3363e-12,\n 1.8656e-12, 8.0652e-12, 6.5362e-13, 5.3644e-12, 9.9291e-14, 1.1674e-13,\n 9.8241e-14, 5.7946e-12, 4.8974e-14, 2.6071e-13, 1.6159e-12, 5.4294e-14,\n 9.7372e-12, 3.3890e-13, 1.3558e-11, 1.2786e-12, 1.2076e-11, 3.1599e-12,\n 2.2731e-12, 7.7653e-13, 4.9527e-12, 1.8637e-12, 2.9461e-12, 8.9375e-14,\n 4.7448e-12, 5.4229e-13, 2.1649e-12, 1.7269e-12, 3.7640e-12, 5.4326e-12,\n 3.0085e-12, 1.3089e-13, 8.5337e-14, 4.3576e-14, 1.9856e-12, 2.5500e-12,\n 3.6182e-13, 2.1028e-12, 3.0968e-13, 1.1157e-12, 9.0452e-12, 2.6439e-13,\n 3.0937e-13, 5.9858e-13, 8.5331e-14, 3.7699e-12, 4.0269e-12, 2.3306e-12,\n 2.8808e-12, 3.1088e-12, 1.9063e-12, 8.2345e-13, 2.2850e-12, 1.0592e-13,\n 1.1440e-11, 7.9965e-13, 2.3144e-12, 5.1724e-12, 1.9864e-11, 2.4405e-11,\n 2.7964e-13, 2.4210e-12, 8.3568e-13, 2.8503e-12, 1.9109e-12, 1.6326e-13,\n 5.8752e-12, 9.1318e-14, 1.9942e-13, 1.5201e-13, 4.8161e-13, 1.1031e-11,\n 3.1452e-13, 6.1915e-14, 6.5686e-13, 4.6695e-12, 2.8380e-12, 6.6130e-13,\n 1.0876e-13, 5.1400e-13, 2.5822e-12, 2.0030e-11, 1.1006e-11, 7.2682e-12,\n 9.0143e-13, 3.2372e-12, 2.5507e-12, 3.4208e-12, 4.8030e-12, 2.6243e-13,\n 8.6602e-12, 1.2993e-13, 3.6767e-12, 3.2318e-12, 7.7620e-13, 3.2819e-12,\n 3.9871e-13, 9.7229e-12, 7.4492e-13, 2.3305e-12, 3.3517e-12, 1.4485e-11,\n 2.2438e-13, 2.3731e-12, 5.0276e-13, 3.7441e-12, 2.6478e-13, 6.3252e-13,\n 7.5609e-14, 1.5586e-12, 2.6333e-14, 2.3091e-12, 5.9269e-13, 1.7301e-13,\n 6.7950e-12, 1.9113e-13, 1.0364e-11, 2.5863e-12, 3.9523e-12, 2.5117e-12,\n 3.3956e-13, 5.5710e-12, 8.7092e-13, 1.9927e-12, 3.7181e-12, 7.0655e-13,\n 1.8726e-12, 6.3924e-13, 9.6511e-12, 1.5460e-12, 2.2788e-13, 6.3140e-14,\n 1.2928e-13, 8.6087e-12, 6.1401e-12, 1.9327e-13, 2.3688e-12, 2.4099e-12,\n 5.3368e-13, 2.3700e-12, 1.0895e-11, 2.1446e-12, 1.0670e-12, 1.1008e-12,\n 1.0882e-13, 1.2401e-12, 6.9070e-12, 4.4174e-12, 2.3078e-13, 4.9746e-12,\n 4.4359e-12, 7.5158e-14, 3.1274e-12, 3.9531e-12, 1.8104e-12, 5.7994e-13,\n 9.5081e-12, 8.1595e-13, 1.2866e-11, 2.7171e-12, 3.1632e-12, 7.4225e-13,\n 1.6547e-13, 1.9484e-13, 2.5127e-13, 3.0875e-12, 2.6405e-12, 2.8865e-12,\n 1.4467e-12, 2.3483e-12, 2.4822e-13, 1.4763e-12, 3.1468e-11, 1.0501e-12,\n 4.3227e-12, 3.9159e-13, 1.4858e-12, 7.7389e-14, 1.6598e-12, 6.0828e-12,\n 7.3156e-13, 1.1340e-13, 1.2509e-12, 7.7402e-13, 1.3973e-12, 1.7866e-12,\n 6.0475e-12, 3.6148e-12, 7.7252e-13, 9.5465e-13, 2.3318e-13, 3.7604e-12,\n 9.3413e-13, 6.5648e-12, 4.6201e-12, 5.8501e-12, 2.1490e-11, 4.5691e-13,\n 5.5024e-12, 2.8752e-13, 3.5787e-13, 4.2769e-12, 3.0203e-12, 2.1519e-13,\n 6.8731e-13, 2.1278e-13, 7.4692e-12, 7.3168e-14], device='cuda:0')" + }, + "44": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-5.0882e-15, 1.4556e-14, 1.8200e-14, 3.7989e-13, -1.9858e-15,\n 3.6302e-15, 1.5902e-14, 9.1620e-15, 2.2500e-14, 3.5936e-15,\n 1.0946e-14, 5.0444e-16, -1.0526e-14, 1.5602e-14, 4.5443e-14,\n 1.0628e-13, -6.8738e-16, 4.7856e-15, 2.2261e-15, -2.9277e-15,\n 9.0576e-15, 2.2486e-14, 1.8806e-15, 6.6342e-16, 8.1807e-15,\n 1.3577e-14, 1.5995e-14, -1.2656e-15, 3.0268e-14, 5.1233e-14,\n 5.4603e-15, 1.2550e-14, -1.6870e-15, 2.4862e-14, 5.7105e-15,\n 6.9138e-16, -2.2102e-15, 2.7741e-13, 5.0446e-14, -5.0999e-15,\n -2.0939e-15, 1.6855e-14, -1.6454e-16, -1.9394e-16, 1.4538e-14,\n 6.3068e-16, -7.2229e-16, 2.6865e-14, -7.0098e-15, -3.4256e-15,\n 1.2383e-14, 4.6994e-15, -3.4989e-15, 1.5541e-14, 3.8835e-14,\n 7.4227e-15, 5.5767e-15, 2.9403e-14, 4.6241e-14, 2.3838e-13,\n 4.5882e-16, 1.1861e-15, 1.5339e-13, 1.2201e-13, 3.9958e-15,\n 3.2707e-14, 1.9664e-15, 2.3692e-14, 3.6979e-15, 3.9392e-14,\n 1.8161e-14, 2.4120e-14, 1.7633e-14, 3.5310e-14, 7.4786e-15,\n -8.2650e-17, 5.0158e-16, -1.4512e-15, 1.0919e-13, 4.6792e-14,\n -4.5169e-15, 6.9246e-15, 3.1861e-13, 1.3902e-13, 1.4023e-14,\n 7.2618e-14, 3.4738e-14, -6.9089e-15, -1.0700e-14, -6.1255e-15,\n 5.5027e-16, 7.9138e-14, -2.8105e-14, -1.0500e-15, -3.0832e-15,\n 1.8255e-13, 8.5956e-14, -8.2013e-17, 3.8601e-14, -5.5014e-15,\n -2.4593e-14, -4.3947e-15, -1.2897e-14, 5.9333e-14, -6.1468e-15,\n 2.2293e-15, -3.2015e-15, 1.6591e-14, 5.2701e-14, 5.4001e-16,\n 1.9086e-14, 2.8720e-14, 9.9998e-15, 8.3349e-15, -1.0223e-14,\n -2.9223e-14, 3.4933e-15, 3.6743e-14, 2.7411e-15, 6.4399e-14,\n -1.1456e-14, 4.5386e-15, 2.3242e-13, 7.9848e-14, 2.7750e-13,\n 1.0517e-14, -4.6194e-16, -3.8740e-15, 5.7781e-14, 1.1961e-14,\n 3.0347e-14, 3.8274e-15, 3.5151e-15, 3.8047e-14, -1.1336e-14,\n -7.7730e-15, -8.1470e-15, -2.4953e-15, 2.9379e-13, 2.2549e-14,\n 2.5065e-14, 1.3438e-14, -1.0828e-13, 2.9555e-15, 4.3313e-15,\n -6.2638e-15, 2.0776e-14, -3.8583e-16, 2.2654e-14, -4.4311e-15,\n -2.3777e-16, 1.5042e-14, -1.2483e-14, 3.1191e-13, 3.3352e-13,\n 2.5160e-15, -4.1462e-15, 1.6517e-15, 4.0259e-16, -4.1405e-15,\n -4.6563e-15, 2.6829e-15, 1.3461e-14, -4.9160e-15, 7.6831e-15,\n 4.2951e-14, 1.1476e-14, 5.6557e-14, -5.7480e-15, 1.2423e-14,\n 3.1185e-14, 1.2023e-13, -4.7168e-14, 9.0623e-15, -1.8465e-14,\n -6.1130e-16, -1.3865e-14, -5.2765e-16, 3.7547e-15, -4.8735e-16,\n 6.9711e-15, 1.8897e-14, 3.5082e-15, -9.2179e-15, 5.3252e-14,\n 3.4852e-15, 9.5725e-14, -6.3857e-16, -1.7676e-15, 3.5859e-16,\n 7.8789e-14, 9.6883e-15, -3.4648e-15, -4.6697e-18, 4.4843e-16,\n 2.4447e-15, -2.6433e-15, 2.3387e-14, -2.1851e-15, -5.7073e-15,\n 1.1633e-14, 5.2934e-16, 2.4120e-13, -5.3469e-15, 2.1199e-13,\n -3.2492e-14, -8.4908e-15, -1.6618e-15, 1.8703e-13, 1.0075e-13,\n -9.1309e-15, 8.0760e-15, 1.0018e-14, 1.8757e-14, 3.3826e-15,\n -1.3555e-14, 6.1434e-14, 1.2054e-14, 1.3110e-13, 3.4480e-14,\n 9.5208e-15, 1.9076e-14, 3.7958e-14, 1.4692e-14, -4.2187e-16,\n 1.0173e-13, -9.1074e-16, 3.0967e-15, 2.3879e-14, 2.8984e-14,\n 1.1011e-13, 1.8279e-14, -3.3650e-15, -1.6212e-15, 2.2821e-15,\n 9.0560e-15, -1.8768e-15, -1.8296e-14, -6.3049e-15, 1.5139e-14,\n 8.2023e-14, 6.0243e-15, 1.9336e-14, 1.8351e-14, 5.7418e-15,\n 3.2256e-15, 1.8231e-14, 5.2227e-15, 5.4218e-15, 1.3342e-14,\n 7.0422e-15, -4.5601e-14, 3.2178e-14, 3.5321e-13, 4.1410e-14,\n 1.1453e-14, 3.3910e-13, -4.2310e-15, -4.2103e-15, 2.2831e-15,\n 4.7764e-15, 3.0248e-13, 3.4839e-14, -9.9643e-15, -1.7828e-15,\n -6.8744e-16, -6.7514e-16, 2.3424e-14, 1.2527e-13, 5.8894e-14,\n 1.0313e-15, 9.0421e-15, -1.4751e-16, 4.1578e-14, -6.3808e-16,\n -8.6533e-17, 3.4869e-16, 4.5207e-14, 1.8146e-15, 2.6003e-15,\n 1.6405e-15, -3.4063e-15, 8.4624e-14, 3.0134e-14, 4.8955e-15,\n 9.9699e-15, 1.9940e-15, -2.1898e-14, -3.5178e-15, -1.0983e-14,\n 1.0641e-13, 2.2556e-14, 1.7362e-14, 1.7793e-14, -1.1886e-14,\n -7.4529e-14, 1.1946e-15, 2.7162e-15, -5.6286e-14, 1.5121e-13,\n -4.1247e-14, -7.4779e-16, -5.0787e-16, 1.2118e-14, 1.0778e-13,\n -6.1993e-15, 2.8517e-15, -9.3267e-14, 1.7594e-13, 3.0005e-13,\n 3.3627e-15, 4.8203e-14, 9.1638e-16, 1.2629e-14, 5.8622e-15,\n 7.0288e-15, 6.5852e-15, 1.7285e-13, -6.6808e-14, 2.8762e-13,\n -7.5842e-15, 1.5840e-13, 1.2407e-14, 7.3609e-15, 9.7278e-15,\n 1.8012e-14, 4.0327e-14, 4.5060e-15, -2.5883e-15, 8.9553e-14,\n 7.9625e-15, -1.7766e-15, -2.7232e-15, 1.7939e-14, -1.6392e-14,\n 2.5135e-15, -1.1327e-14, -2.6765e-15, 1.5216e-13, -4.0964e-15,\n -5.5107e-16, -1.1858e-14, 4.4169e-14, 2.1054e-15, 1.2826e-14,\n 5.4886e-16, 2.3980e-14, -3.9385e-15, -1.3192e-15, 5.9781e-15,\n -1.2073e-15, 1.0728e-14, 9.6681e-14, -4.4714e-15, 1.0471e-13,\n -1.0151e-14, 7.7646e-14, -1.9643e-14, 1.1221e-13, 8.2057e-15,\n -4.8947e-15, -1.0096e-14, 1.1964e-14, -3.7920e-15, 1.5533e-13,\n 1.0333e-14, -4.3241e-16, -1.4403e-15, 3.8579e-15, 1.8506e-14,\n 1.2469e-14, 1.4137e-14, 3.5002e-15, 8.0553e-14, 3.5955e-14,\n 2.4377e-13, 2.8044e-15, -1.2542e-15, 7.9385e-15, 4.8057e-14,\n 2.3389e-14, 9.8388e-15, 6.0594e-14, 1.1912e-15, -7.0165e-15,\n -1.7986e-14, 1.2116e-13, 5.7677e-14, -2.0667e-15, 1.3813e-13,\n 8.8061e-16, 1.4332e-13, -2.6661e-14, -3.0697e-15, -5.1483e-15,\n 3.7358e-13, -2.7824e-15, 6.9306e-15, 2.5695e-16, 4.3062e-15,\n 1.1824e-13, -2.5747e-15, 2.0990e-13, 4.5294e-14, -8.0797e-15,\n 1.7102e-15, -7.4597e-16, 2.1693e-14, -2.7112e-15, 1.2164e-13,\n 1.3021e-14, -7.4969e-16, 5.1787e-15, 2.5525e-15, -1.6588e-15,\n 6.0568e-14, 4.0626e-14, 9.7727e-14, 3.0241e-13, 9.0754e-16,\n -1.2938e-15, 2.6482e-14, -2.1789e-16, -1.3119e-14, -6.4074e-15,\n 2.0778e-14, 5.5930e-15, 2.0443e-14, -2.6093e-15, -1.0602e-15,\n 8.7223e-14, 6.7290e-15, 7.8925e-14, -1.0487e-14, 3.8664e-15,\n 6.4575e-14, 3.6079e-15, 1.1918e-14, -3.5589e-15, -1.1267e-14,\n -6.4636e-15, 1.0075e-15, -1.1539e-16, 8.0797e-15, 7.9265e-14,\n -8.1134e-16, -6.8635e-14, 3.7431e-15, 3.5075e-14, -7.1908e-15,\n -3.0839e-16, 2.8748e-14, 7.7254e-14, -1.7611e-15, 5.9957e-14,\n 3.2579e-15, 4.0325e-14, 1.6423e-13, 1.9742e-15, -3.1397e-15,\n 7.2800e-14, -1.0626e-14, -3.5909e-15, 2.5613e-14, 1.1721e-13,\n -5.6952e-19, -3.4630e-15, 9.4536e-15, 1.5031e-15, -7.1261e-15,\n -1.8050e-15, 6.8740e-14, -2.2701e-15, -5.2550e-17, 2.7122e-15,\n -9.7421e-16, -7.6071e-15, -2.5399e-15, 1.2537e-14, -2.0299e-15,\n 6.1453e-14, -4.5186e-14, 4.5976e-14, -2.3548e-14, 1.7580e-14,\n 9.8448e-15, 4.9205e-14, 1.6406e-14, 1.4370e-13, 3.9498e-14,\n 1.1635e-13, 1.2785e-13, -2.9121e-15, 3.1279e-14, 2.4396e-14,\n 2.7723e-13, 1.6968e-15, 4.9879e-15, -1.1256e-13, 9.3343e-16,\n -6.8426e-17, 1.8088e-14, -4.0983e-15, 2.6815e-14, 4.2772e-14,\n 1.6659e-15, -2.8470e-15, 9.4154e-15, -8.5982e-16, -3.8070e-15,\n 2.7470e-15, -5.6801e-15, 5.0143e-14, -8.3579e-15, -9.5519e-15,\n 9.5278e-15, -5.8547e-15, -4.8746e-14, 3.5968e-14, 1.0303e-13,\n 1.4348e-13, 7.2151e-15, 1.7288e-14, 1.5399e-14, -1.2984e-15,\n 2.1109e-14, 3.5398e-15, 8.1991e-15, -1.7350e-15, 5.9145e-15,\n -2.4684e-15, 1.6219e-13, 3.0929e-14, 5.1739e-14, 1.1714e-14,\n 1.3726e-14, -1.0907e-14, 1.2837e-13, 8.8396e-15, -2.8133e-15,\n 9.6625e-15, 1.5943e-14, 2.9059e-14, 4.8620e-14, 2.6952e-14,\n 6.4267e-16, 1.5236e-13, -2.3878e-15, 7.5829e-14, 1.1995e-15,\n 1.5462e-13, 2.6926e-14, 5.5926e-15, 7.8735e-14, -2.6695e-15,\n 3.8428e-15, 8.5961e-15, 4.0898e-14, 5.9572e-15, 9.6638e-14,\n -1.5919e-14, 2.1494e-15, 1.7981e-14, -2.0139e-15, 8.4664e-16,\n 1.1135e-13, -4.2933e-15, 5.7710e-15, -8.2781e-15, 1.4550e-13,\n 1.9362e-14, -6.6803e-15, 4.1347e-15, -5.6981e-15, -1.7335e-15,\n 5.3327e-15, 5.1842e-16, 1.5771e-14, 3.2224e-14, 1.2371e-13,\n -1.1372e-15, 2.6973e-14, 3.1120e-14, 1.1634e-14, 1.8153e-14,\n 1.7730e-13, 1.0408e-14, 1.2221e-13, 1.0180e-14, 1.5124e-14,\n 4.3780e-14, -2.3263e-15, 5.0392e-14, 6.9514e-15, 1.6209e-13,\n 1.9624e-14, 6.2586e-15, 3.5106e-14, -2.8151e-15, 2.6843e-14,\n 5.6235e-15, 1.5302e-13, -3.8514e-15, -1.4321e-14, -5.5119e-16,\n -4.4493e-16, -2.8796e-14, 1.7514e-15, 1.4935e-14, 1.3865e-14,\n -1.1268e-15, -3.3380e-14, 1.8070e-14, 7.0031e-15, 2.8964e-17,\n -2.4123e-15, -4.1957e-16, 7.3465e-15, 5.2772e-14, 2.4360e-13,\n 1.0218e-14, -8.9454e-16, 3.1432e-14, 4.2457e-14, 7.0238e-14,\n -9.7509e-15, -1.0179e-14, 4.0452e-14, 7.4263e-14, 1.0611e-13,\n 6.1660e-14, 1.3960e-14, 2.9666e-15, 1.5639e-13, 8.4301e-15,\n 3.6972e-15, -4.2559e-15, 2.5380e-14, -9.1251e-14, 2.0885e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.2186e-15, 1.9247e-15, 4.5082e-16, 2.3780e-16, 1.5703e-17, 2.5844e-17,\n 1.4148e-15, 6.0760e-18, 5.8190e-17, 6.9657e-17, 2.2994e-16, 8.1571e-16,\n 1.4726e-15, 1.4286e-15, 1.1756e-15, 1.5457e-15, 1.7964e-15, 6.5911e-17,\n 2.2294e-15, 4.5467e-17, 6.9731e-18, 8.8805e-16, 1.8314e-17, 7.5902e-18,\n 7.7376e-15, 1.6397e-16, 1.4597e-16, 8.5665e-17, 1.7584e-15, 1.1525e-15,\n 2.5219e-17, 3.0657e-16, 2.2159e-15, 2.4883e-15, 9.5359e-17, 8.6578e-16,\n 4.9166e-15, 5.0640e-17, 1.1661e-16, 6.8756e-18, 2.7282e-16, 3.7929e-17,\n 2.7261e-17, 4.3749e-16, 4.7596e-17, 2.8467e-15, 1.1645e-14, 1.5718e-15,\n 1.2518e-16, 2.1324e-15, 5.2744e-17, 1.6620e-17, 8.6021e-17, 4.1539e-17,\n 9.1440e-15, 2.0653e-17, 6.9264e-18, 7.0851e-17, 4.7363e-15, 7.6210e-15,\n 2.9126e-16, 5.9751e-16, 1.9433e-17, 4.5377e-14, 6.2451e-17, 1.8468e-17,\n 9.3199e-16, 6.6752e-16, 1.0307e-16, 2.3575e-15, 2.1923e-15, 2.1533e-17,\n 1.2364e-15, 2.7957e-15, 1.9824e-17, 1.6481e-16, 3.4564e-16, 1.9582e-16,\n 2.5581e-14, 1.1482e-15, 1.2730e-16, 3.2481e-16, 1.0826e-15, 5.4957e-16,\n 2.9356e-17, 4.0271e-15, 7.8624e-16, 2.1394e-17, 2.3098e-16, 1.6622e-15,\n 5.6663e-17, 3.5610e-16, 6.0320e-17, 1.4372e-17, 3.8011e-17, 1.7390e-16,\n 1.9230e-15, 4.2108e-17, 1.2114e-15, 1.6248e-17, 4.0222e-17, 2.6132e-16,\n 7.7180e-15, 4.4306e-16, 8.1097e-15, 1.7311e-16, 7.9685e-18, 5.3535e-16,\n 3.0369e-17, 4.4534e-16, 6.3203e-16, 9.9973e-15, 2.1875e-16, 7.3651e-17,\n 4.4885e-17, 1.1256e-16, 1.2090e-16, 2.5453e-15, 7.8000e-18, 1.4382e-14,\n 6.8241e-16, 5.8912e-17, 7.1359e-17, 2.7856e-16, 1.6949e-17, 1.5580e-14,\n 1.0550e-16, 3.1186e-17, 1.1868e-15, 1.9186e-17, 1.7866e-15, 5.2948e-17,\n 1.6996e-15, 1.2810e-15, 2.3169e-16, 2.4031e-16, 1.5347e-16, 7.2842e-17,\n 7.3490e-16, 1.5858e-15, 3.8991e-17, 1.4254e-16, 3.1973e-15, 1.3137e-17,\n 1.1910e-16, 8.4368e-17, 1.0252e-16, 7.6264e-16, 2.8550e-17, 3.0452e-16,\n 9.0943e-17, 8.5468e-18, 7.0313e-15, 3.5502e-15, 1.7383e-15, 8.1283e-17,\n 3.2043e-16, 7.3497e-17, 1.2835e-16, 4.6373e-16, 1.0730e-16, 3.0505e-16,\n 4.2790e-18, 3.2779e-15, 1.4206e-16, 1.2674e-15, 1.3156e-17, 4.5168e-15,\n 1.8202e-15, 1.1914e-15, 7.7246e-17, 9.9544e-16, 9.1596e-16, 9.4447e-17,\n 9.7044e-15, 2.2285e-15, 4.4150e-15, 7.1162e-17, 2.8096e-16, 2.7501e-17,\n 2.6407e-15, 1.6213e-16, 1.8227e-16, 1.5581e-15, 8.5931e-16, 2.6901e-17,\n 3.8980e-17, 6.9955e-18, 3.4761e-15, 8.7944e-18, 1.4695e-16, 1.1306e-17,\n 1.3791e-15, 1.1105e-16, 3.0668e-15, 1.2483e-16, 6.5674e-17, 1.1733e-15,\n 3.5622e-15, 2.5588e-15, 1.1830e-15, 3.7489e-16, 1.5090e-14, 3.5987e-17,\n 1.2634e-15, 6.2576e-15, 1.5026e-14, 3.6650e-17, 1.9181e-16, 5.8250e-15,\n 3.7277e-17, 9.6946e-17, 1.2080e-16, 7.1461e-18, 3.7051e-16, 4.4005e-16,\n 7.2337e-16, 1.7859e-15, 3.0252e-15, 1.6132e-15, 1.0412e-17, 4.4111e-17,\n 4.9398e-15, 1.5800e-14, 2.1483e-18, 3.0410e-14, 3.7704e-15, 2.6895e-16,\n 7.4429e-18, 1.1291e-14, 2.3773e-15, 6.4095e-16, 7.8622e-16, 1.3307e-15,\n 1.0505e-16, 1.6937e-16, 2.6110e-15, 2.4008e-17, 1.4936e-15, 1.5009e-15,\n 9.1121e-16, 2.1524e-16, 6.8676e-17, 1.5571e-15, 9.5037e-16, 7.9358e-18,\n 6.4083e-15, 1.1640e-16, 1.2326e-16, 1.3109e-17, 3.6662e-16, 8.1276e-15,\n 4.4541e-16, 1.5180e-14, 2.5132e-15, 3.5879e-16, 1.7342e-15, 3.7845e-15,\n 1.6004e-15, 2.3293e-17, 4.3999e-17, 7.3452e-15, 3.9306e-16, 1.3369e-16,\n 2.7739e-18, 1.0991e-16, 1.1777e-17, 1.4067e-15, 2.2700e-15, 4.5559e-18,\n 5.2047e-17, 3.1957e-17, 2.1827e-16, 6.3185e-15, 6.8166e-17, 2.3934e-15,\n 1.4188e-17, 2.7954e-15, 4.9024e-17, 1.2226e-16, 8.3942e-16, 3.7534e-17,\n 4.2544e-17, 2.0437e-16, 3.0188e-17, 9.3000e-18, 3.4151e-18, 1.6473e-15,\n 5.9230e-17, 3.4132e-15, 2.5568e-16, 1.4735e-14, 1.5448e-16, 3.9372e-15,\n 1.1723e-14, 1.0270e-16, 5.4398e-15, 1.2458e-15, 6.8568e-16, 1.7137e-15,\n 1.9227e-17, 5.4099e-16, 5.8744e-16, 1.8494e-16, 7.4656e-15, 4.6087e-15,\n 2.6037e-14, 8.3746e-15, 2.0669e-15, 1.5301e-15, 1.2447e-16, 3.5409e-16,\n 4.0834e-16, 4.3978e-17, 2.4250e-16, 7.1183e-17, 9.7473e-18, 1.8862e-15,\n 1.3082e-15, 1.1590e-14, 1.3331e-15, 4.8211e-17, 1.9152e-15, 8.4657e-18,\n 1.6525e-15, 5.9832e-17, 9.3915e-15, 5.1835e-17, 1.3450e-16, 3.2408e-16,\n 4.1964e-17, 1.2445e-15, 3.4299e-17, 7.6736e-17, 1.1415e-14, 1.0530e-16,\n 3.9060e-16, 1.6934e-17, 3.7671e-17, 7.2698e-17, 5.1360e-17, 3.7443e-16,\n 1.1357e-14, 5.5986e-17, 5.6089e-17, 4.8746e-17, 6.8244e-18, 6.2813e-15,\n 2.1713e-17, 7.7496e-15, 1.1503e-16, 5.0849e-15, 3.5420e-17, 2.7576e-16,\n 1.6547e-14, 1.2085e-15, 1.2461e-16, 3.6435e-15, 7.1032e-17, 8.9709e-18,\n 6.9983e-16, 5.1668e-16, 3.0558e-16, 5.0862e-17, 5.1952e-16, 7.9490e-18,\n 6.0620e-16, 2.2710e-15, 4.0031e-16, 8.5525e-16, 2.8603e-17, 1.0920e-16,\n 5.0095e-18, 4.8006e-15, 6.3981e-15, 1.5375e-14, 1.6323e-15, 1.8096e-15,\n 1.1913e-16, 3.1270e-16, 1.9505e-16, 8.5343e-16, 3.5511e-16, 7.5268e-18,\n 1.6644e-17, 8.4621e-16, 1.3940e-15, 4.3354e-16, 5.3027e-15, 7.8151e-16,\n 5.5252e-16, 2.6668e-15, 3.0294e-15, 4.6125e-17, 1.8689e-16, 5.6100e-18,\n 1.9274e-17, 1.9198e-16, 3.6041e-16, 5.4478e-16, 1.2684e-16, 1.2714e-16,\n 1.7312e-16, 2.2197e-15, 5.7380e-16, 5.1730e-17, 3.5254e-18, 3.3173e-15,\n 3.3415e-17, 2.9967e-16, 4.4948e-16, 3.2553e-15, 6.4373e-16, 7.9386e-17,\n 2.0695e-16, 3.2339e-15, 3.6544e-17, 2.3882e-17, 1.2846e-16, 3.4150e-16,\n 4.9023e-16, 4.5016e-17, 6.1063e-17, 3.5874e-16, 4.5997e-17, 1.0805e-16,\n 2.3796e-16, 1.4393e-15, 4.3542e-16, 2.8090e-16, 3.2156e-15, 1.6001e-15,\n 1.0517e-15, 1.7334e-16, 5.2818e-16, 3.0417e-15, 1.1015e-16, 6.9480e-17,\n 2.5753e-17, 3.6805e-15, 3.3113e-17, 2.2030e-16, 1.7995e-16, 8.0631e-18,\n 1.8377e-15, 7.7694e-17, 1.6775e-14, 8.8084e-18, 3.6555e-15, 8.1469e-17,\n 8.1368e-16, 3.6575e-16, 4.1094e-15, 6.1506e-16, 7.5900e-16, 6.4622e-17,\n 1.0326e-14, 8.8270e-17, 1.3390e-16, 3.2728e-17, 1.9772e-16, 3.4463e-15,\n 1.3305e-15, 5.3824e-17, 1.9141e-17, 4.9544e-17, 3.1877e-16, 5.4488e-16,\n 2.8067e-17, 1.2062e-15, 2.6116e-17, 1.1926e-15, 1.9031e-15, 2.7608e-17,\n 3.3598e-17, 2.0016e-16, 1.1600e-17, 1.6743e-15, 1.5774e-15, 1.1597e-15,\n 4.3266e-15, 1.5825e-15, 1.0437e-15, 8.8092e-18, 6.8110e-17, 5.3861e-18,\n 1.4164e-14, 4.1770e-16, 3.4824e-15, 2.3048e-15, 1.7257e-14, 1.7193e-14,\n 3.5636e-17, 7.0736e-16, 2.3501e-16, 2.1529e-15, 2.5601e-15, 3.1672e-17,\n 4.9255e-15, 7.0877e-17, 7.1632e-17, 5.3776e-17, 4.2126e-17, 7.9746e-15,\n 4.5309e-18, 1.5246e-17, 9.4577e-17, 4.0955e-15, 6.2582e-16, 1.5827e-16,\n 1.6542e-16, 6.0064e-17, 4.6904e-16, 7.2197e-15, 1.3178e-15, 3.5114e-15,\n 9.5595e-17, 2.0226e-16, 3.3529e-15, 4.5968e-15, 2.0043e-15, 3.7216e-17,\n 8.6176e-15, 4.0475e-17, 2.0339e-15, 1.4005e-15, 1.0190e-16, 2.3467e-16,\n 1.3759e-17, 3.1331e-15, 4.9565e-16, 1.3744e-15, 2.8706e-16, 1.3759e-14,\n 1.2844e-16, 1.1243e-15, 4.8261e-17, 7.7769e-16, 1.6991e-17, 2.5750e-17,\n 2.2308e-17, 5.3777e-16, 1.4403e-17, 1.7000e-15, 6.4305e-17, 5.3494e-18,\n 9.8347e-15, 9.8238e-17, 1.2217e-14, 1.0251e-15, 3.8645e-15, 2.1274e-15,\n 8.0058e-17, 3.7330e-15, 3.5238e-17, 2.2682e-15, 6.5185e-16, 1.5301e-17,\n 4.7497e-16, 3.1249e-17, 4.5777e-15, 1.3950e-15, 1.1684e-17, 7.5260e-17,\n 3.6655e-17, 4.5581e-15, 3.6448e-15, 3.1755e-17, 4.6611e-16, 7.4876e-16,\n 5.0138e-17, 8.2401e-16, 2.4119e-15, 3.0231e-16, 2.4328e-17, 2.5535e-16,\n 3.0073e-17, 6.0983e-17, 5.1244e-15, 1.5073e-15, 3.9602e-17, 6.0698e-15,\n 3.3711e-15, 1.3155e-17, 1.0529e-15, 9.5896e-16, 8.3337e-16, 3.8737e-17,\n 1.3697e-14, 1.8146e-16, 1.5634e-14, 7.0959e-16, 8.6296e-16, 1.7262e-16,\n 2.0586e-17, 1.7157e-16, 3.2016e-17, 1.9166e-15, 5.0338e-16, 9.4048e-16,\n 9.3249e-16, 1.4626e-15, 6.6849e-17, 4.1636e-16, 5.9517e-15, 2.1034e-16,\n 1.8624e-15, 7.6466e-18, 2.4460e-16, 1.3257e-17, 7.3896e-16, 5.0365e-15,\n 3.2148e-17, 1.3804e-17, 4.7779e-17, 7.5059e-17, 2.5500e-16, 2.1798e-15,\n 2.8475e-15, 3.4223e-15, 1.9081e-16, 1.8080e-16, 4.5695e-18, 4.2203e-16,\n 3.8417e-16, 3.8702e-15, 1.2806e-15, 2.2749e-15, 9.5778e-15, 3.9736e-17,\n 3.1158e-15, 2.7701e-17, 3.9749e-17, 3.3027e-15, 1.3275e-15, 1.2542e-16,\n 7.1176e-16, 2.5270e-17, 4.6429e-15, 1.2187e-17], device='cuda:0')" + }, + "45": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.8343e-14, 5.8857e-14, 1.6373e-13, 1.9649e-13, -1.1777e-15,\n -2.2884e-15, 9.0628e-15, -9.4083e-15, -1.6787e-14, -3.4851e-15,\n 1.1739e-13, -4.0249e-15, 5.1850e-14, 2.0524e-14, 1.4706e-13,\n 1.3619e-13, 6.8899e-14, -2.9694e-15, 7.5520e-14, 7.2152e-15,\n -1.0200e-14, 8.8540e-14, -9.3249e-16, 1.1408e-14, 7.5216e-14,\n 5.8288e-14, -1.2276e-14, 1.2162e-15, -2.0461e-14, 8.2094e-14,\n -4.9733e-15, 8.6059e-14, 6.4520e-15, -1.5598e-14, 6.9014e-14,\n -6.0495e-15, 2.0498e-14, 1.7754e-13, 1.2862e-13, 4.4805e-14,\n 5.7853e-14, -1.3268e-14, 1.5931e-14, -1.2607e-14, -1.2353e-14,\n -2.8982e-15, 3.8550e-14, 9.8169e-14, 2.6084e-14, 8.2820e-15,\n -1.2107e-14, -4.0859e-15, 5.6625e-14, 4.9808e-14, 1.0452e-13,\n -3.4077e-15, 6.9805e-14, -2.3851e-14, 7.3718e-14, 1.4786e-13,\n 1.2807e-16, -8.9349e-16, 1.2189e-13, 1.1429e-13, -3.6899e-15,\n 6.9995e-14, -3.8125e-15, -1.9416e-14, 1.8012e-14, 1.2037e-13,\n -1.7209e-14, -2.1112e-14, 4.0401e-14, 6.6872e-14, -3.5447e-15,\n -6.8452e-15, -3.9788e-16, 1.9465e-14, 1.0167e-13, 8.3150e-14,\n 1.0524e-14, -7.3386e-15, 2.6623e-13, 1.2154e-13, -5.4596e-15,\n 8.8160e-14, 8.3110e-14, 1.2930e-14, -7.4140e-15, 1.6530e-14,\n 9.9917e-16, 9.9305e-14, -4.9731e-14, 1.0017e-15, 5.0310e-14,\n 9.5744e-14, 6.0998e-14, 3.4386e-15, 4.4339e-14, 2.0114e-14,\n -1.8607e-14, -7.6724e-15, 4.6922e-14, 1.0871e-13, 4.2978e-14,\n -1.7368e-15, 1.3479e-14, 5.8368e-14, 4.4806e-14, 2.3967e-16,\n 1.4062e-14, 7.5425e-14, 4.2702e-14, 4.5326e-14, 1.7759e-14,\n -1.8128e-14, 2.6379e-14, 7.7850e-14, 1.7528e-14, 1.0442e-13,\n 5.4083e-14, -2.0298e-15, 1.5930e-13, 1.1535e-13, 2.1837e-13,\n 4.7652e-15, 8.6058e-16, 2.9423e-15, 6.0468e-14, 5.3654e-14,\n 4.9225e-14, -2.3372e-15, 5.5138e-14, 4.3477e-14, 4.5104e-14,\n 4.1400e-14, 5.4801e-14, 2.2715e-15, 2.3761e-13, 7.4173e-14,\n -1.4513e-14, -1.1535e-14, -9.8691e-14, -3.5488e-15, -2.6011e-15,\n 6.2711e-14, 3.4742e-14, 3.0156e-15, -1.2634e-14, 1.9814e-14,\n 4.6628e-14, -6.6241e-15, 2.4822e-14, 1.9288e-13, 1.9459e-13,\n 3.9375e-14, 6.3581e-15, -2.0599e-15, -4.7892e-16, 3.5407e-14,\n 9.8086e-15, -2.4860e-15, -6.2374e-15, 6.6006e-14, -4.5389e-15,\n 1.4649e-13, -9.7855e-15, 7.3426e-14, 2.2628e-14, 3.2228e-14,\n -2.1997e-14, 1.5912e-13, -5.5425e-14, -3.6230e-15, 8.4229e-14,\n 4.4555e-16, 3.4118e-14, -9.5838e-15, 2.0489e-14, 1.9568e-15,\n 6.0812e-14, -8.0682e-15, -3.0039e-15, 2.3976e-14, 5.4831e-14,\n -3.0702e-15, 6.4719e-14, -2.5221e-14, 6.1319e-15, -1.8210e-15,\n 9.4280e-14, 7.6345e-14, 3.0772e-14, -1.1034e-16, 2.9615e-15,\n 9.9585e-15, 5.4269e-15, 7.3686e-14, 2.1089e-15, 2.6150e-14,\n 4.1429e-14, -3.7852e-15, 9.0952e-14, -6.5028e-15, 1.1269e-13,\n -1.1509e-13, 7.3771e-14, -4.5827e-15, 1.8827e-13, 6.0287e-14,\n 2.3147e-14, 2.6096e-14, -7.0470e-15, 5.3149e-14, 6.1805e-14,\n 5.7574e-14, 9.7910e-14, 1.2141e-13, 6.9943e-14, -2.0622e-14,\n 1.8222e-14, -1.5922e-14, 5.8105e-14, 2.5079e-14, 9.2101e-15,\n 1.0345e-13, 4.3800e-16, 1.2007e-14, 1.0937e-13, 6.9918e-14,\n 1.3869e-13, 8.7331e-14, 1.1288e-14, 5.2133e-14, 1.9849e-14,\n 7.3294e-15, 6.2269e-15, -2.4444e-14, 1.5754e-14, 7.2331e-14,\n 1.3756e-13, 3.3645e-14, 4.1606e-14, 1.6890e-14, 6.1219e-14,\n 5.6601e-14, 8.3062e-14, 2.7085e-14, -5.4938e-15, -1.6242e-14,\n 3.7397e-14, -4.7956e-14, -2.2404e-14, 3.3738e-13, 1.0888e-13,\n 6.4635e-14, 1.9863e-13, 3.4928e-14, 1.4782e-14, -1.1124e-15,\n -3.5310e-15, 1.2041e-13, 8.2195e-14, 1.9710e-14, 2.0904e-15,\n 8.1441e-16, -3.1335e-15, 1.1782e-13, 1.4662e-13, 1.3031e-13,\n -1.0513e-16, 6.3776e-14, -8.0902e-16, 4.6975e-14, 2.6256e-15,\n -4.7500e-16, -3.3533e-15, 3.3928e-14, -4.7720e-15, -2.8405e-15,\n -6.4219e-15, 3.9822e-15, 9.8342e-14, 2.6205e-14, -3.7750e-15,\n -7.2668e-15, -1.0024e-15, -2.6263e-14, 1.1608e-15, 6.3725e-14,\n 4.2454e-14, 1.3912e-14, 2.2422e-14, 6.9258e-14, -6.3852e-15,\n -7.4459e-14, 5.1980e-14, 1.8033e-15, -5.1088e-14, 7.5000e-14,\n -7.2565e-14, 2.2756e-15, -5.1214e-15, 2.5305e-14, 6.4020e-14,\n 1.4209e-14, 2.8318e-15, -8.5954e-14, 1.2190e-13, 1.5287e-13,\n -2.3377e-14, 1.3921e-13, -1.2984e-15, 1.1458e-13, 1.5448e-14,\n -5.8626e-15, 2.2502e-14, 1.7404e-13, -4.5668e-14, 2.3051e-13,\n 1.6553e-14, 1.3654e-13, 2.6277e-14, -4.0558e-15, 8.0687e-14,\n -1.5417e-14, 5.3108e-14, -4.6897e-15, 3.3384e-15, 2.2089e-13,\n 1.0466e-13, 6.7873e-14, 1.4087e-14, 2.9174e-14, -3.0382e-14,\n 1.6888e-15, 2.7720e-14, 4.2611e-15, 1.4107e-13, 1.9083e-14,\n 4.3137e-15, 4.6553e-14, 2.6087e-14, -1.4734e-15, 2.3354e-14,\n -3.7295e-16, 2.6191e-14, 6.4861e-14, 1.6476e-14, 5.5292e-14,\n 8.6489e-15, -9.2041e-15, 8.8352e-14, 6.3633e-15, 7.8033e-14,\n 6.1966e-14, 1.0826e-13, -2.3093e-14, 1.2079e-13, 5.1473e-14,\n 3.1415e-14, 7.5486e-14, 3.0647e-14, 1.5483e-15, 1.9476e-13,\n -5.3663e-15, 1.2346e-15, 2.3331e-15, 1.2006e-14, 6.0719e-14,\n -1.1309e-14, -9.5542e-15, 3.2567e-14, 1.3370e-13, 2.8820e-14,\n 1.8294e-13, -1.6615e-15, 1.1196e-14, -5.5944e-15, 5.2077e-14,\n 1.9741e-14, -7.7756e-15, 1.2112e-13, 2.9068e-15, 1.7318e-14,\n -1.4781e-14, 9.5614e-14, 1.0754e-13, 8.7852e-15, 2.3430e-13,\n 4.4365e-14, 1.1733e-13, -5.6129e-14, 7.8278e-15, 8.6663e-15,\n 2.8102e-13, 1.6234e-15, -5.2537e-15, -5.1817e-16, -3.9202e-15,\n 1.3417e-13, 2.8767e-14, 1.1611e-13, 4.1691e-14, 1.2201e-14,\n 4.5680e-14, 1.3230e-15, 2.0851e-14, 1.3675e-15, 1.5514e-13,\n -1.2051e-14, 9.7923e-14, 1.0605e-15, -1.9956e-15, 1.4940e-14,\n 2.8748e-14, -2.2458e-14, 1.2570e-13, 1.7105e-13, 1.8627e-14,\n 3.1222e-15, -1.3493e-14, 3.4151e-16, 2.6809e-14, 5.0369e-14,\n -1.3189e-14, 2.5190e-14, 4.0203e-14, 3.5144e-14, 3.5836e-16,\n 7.1575e-14, 4.3521e-14, 8.4042e-14, 2.3136e-14, 4.0839e-14,\n 7.7180e-14, -2.9249e-15, -1.2823e-14, 3.2842e-14, -1.2669e-14,\n 8.1276e-15, -6.2071e-16, 9.3832e-15, 9.8535e-15, 1.2891e-13,\n 1.5146e-14, -7.7731e-14, -3.8016e-15, 8.9250e-14, 2.6054e-14,\n 1.9937e-15, 3.3034e-14, 7.0959e-14, 1.0464e-15, 4.5298e-14,\n -2.6007e-15, 9.4660e-14, 1.1174e-13, 2.3981e-15, 7.9099e-15,\n 1.1724e-13, 6.0795e-14, -3.8964e-15, -1.5253e-14, 7.1800e-14,\n -2.3260e-17, -1.0889e-14, -6.9240e-15, -1.4211e-15, -1.1771e-14,\n 1.8120e-15, 1.0621e-13, -3.9515e-15, -1.9128e-16, 5.7924e-14,\n 1.3817e-14, 6.0386e-15, 8.6369e-15, 2.7778e-14, 4.9828e-15,\n 1.0239e-13, -5.6718e-14, 1.3654e-13, -4.5209e-14, -1.9451e-14,\n -5.8732e-15, 1.8256e-13, 7.4013e-14, 1.9373e-13, 4.3457e-14,\n 2.3412e-13, 1.7886e-13, 7.7482e-15, 1.0169e-13, 9.1908e-14,\n 1.1848e-13, 4.2176e-15, -4.8281e-15, -8.1096e-14, -8.8879e-16,\n 4.7194e-17, -9.0404e-15, 8.6280e-14, 4.6901e-14, 4.0551e-14,\n -1.9805e-15, 9.3170e-15, 3.1315e-14, 2.8206e-16, 2.1576e-14,\n -1.6317e-15, 1.1691e-14, 5.6876e-14, -3.6225e-14, -1.2205e-14,\n -6.8538e-15, 2.7934e-14, -4.4072e-14, 6.8024e-14, 9.4120e-14,\n 1.1126e-13, -6.9192e-15, 1.2148e-14, -9.5747e-15, 1.3039e-15,\n 2.4203e-14, -1.1622e-14, -7.2106e-15, 2.3833e-15, 1.1725e-14,\n 1.2015e-14, 1.8999e-13, 2.6906e-14, 1.7867e-13, -8.8303e-15,\n 5.6311e-14, 4.1153e-14, 1.2099e-13, -4.5458e-15, 3.4152e-14,\n -8.3897e-15, 2.9138e-14, 4.4774e-14, 6.2010e-14, 2.5717e-14,\n 2.6594e-14, 1.4862e-13, 5.1329e-15, 9.6279e-14, -2.0479e-15,\n 1.5403e-13, -1.8596e-14, 2.0673e-14, 4.8840e-14, -1.0986e-14,\n 2.8193e-14, 1.9087e-14, 3.5657e-14, -5.1661e-15, 1.4254e-13,\n 6.3252e-14, -1.6497e-15, -7.9552e-15, 6.6504e-15, -9.8335e-16,\n 5.9584e-14, 1.2326e-14, -5.1544e-15, 5.4774e-14, 1.1683e-13,\n -1.0816e-14, 3.8460e-14, 1.4019e-14, 1.1880e-14, 1.8644e-15,\n -5.5909e-15, -1.0629e-15, -9.7160e-15, 1.0183e-13, 1.6470e-13,\n 9.9227e-15, 2.3861e-14, 9.6536e-14, 7.6261e-14, 5.8140e-14,\n 1.0112e-13, 1.1655e-14, 1.3686e-13, 8.9234e-15, -1.1008e-14,\n 4.6474e-14, 2.0403e-14, 8.1774e-14, 2.9922e-14, 1.0661e-13,\n -1.6296e-14, -2.4580e-15, 1.2368e-13, 4.5142e-14, 8.0203e-14,\n 1.6858e-14, 1.2586e-13, 7.4529e-15, 3.4742e-14, 1.4676e-15,\n -1.6318e-16, -3.1240e-14, 2.7497e-15, 1.9429e-14, -8.8905e-15,\n 6.6213e-15, -1.5782e-14, 1.6646e-14, -6.9539e-15, -2.5371e-15,\n 3.9592e-15, 4.0151e-16, 2.9502e-15, 3.7595e-14, 1.5544e-13,\n 1.1705e-14, 4.2530e-14, -2.0538e-14, -3.0869e-14, 5.3377e-14,\n -8.0518e-15, 6.6880e-14, 5.9010e-14, 1.6846e-13, 7.0090e-14,\n 1.1698e-13, 7.3908e-14, -6.6603e-15, 1.3190e-13, 2.8868e-14,\n -3.3795e-15, 1.7212e-14, 4.4390e-14, -6.2429e-14, -1.2490e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.2061e-15, 2.9957e-15, 1.2386e-15, 1.3436e-15, 1.5074e-16, 6.5117e-18,\n 1.2977e-15, 5.5238e-17, 1.1959e-17, 1.7296e-16, 2.3927e-15, 1.6025e-15,\n 1.8165e-15, 1.3172e-15, 1.1541e-15, 1.6004e-15, 1.7256e-15, 7.5682e-16,\n 1.3115e-15, 5.5387e-16, 7.6407e-17, 1.0144e-15, 4.7222e-18, 1.3538e-16,\n 2.7628e-15, 9.4482e-16, 5.1739e-17, 6.9808e-16, 3.1380e-15, 1.6278e-15,\n 1.8344e-17, 2.3141e-15, 3.5913e-15, 1.4091e-15, 4.1755e-16, 1.3901e-15,\n 4.5924e-15, 2.8438e-16, 1.7392e-15, 1.3148e-16, 3.0329e-16, 6.7207e-16,\n 2.2466e-16, 1.3629e-15, 2.1729e-17, 2.5374e-15, 1.1907e-14, 3.7045e-15,\n 1.0527e-15, 2.2581e-15, 3.8425e-17, 3.2588e-18, 2.0382e-16, 3.7761e-16,\n 1.4654e-14, 4.5387e-18, 1.7589e-16, 2.2483e-17, 4.0180e-15, 2.8421e-15,\n 1.4401e-15, 2.4209e-15, 7.1647e-16, 9.0328e-15, 2.1466e-17, 1.3095e-17,\n 1.6981e-15, 1.8691e-15, 6.8546e-16, 1.5398e-15, 2.0307e-15, 6.2230e-18,\n 3.5090e-15, 2.7453e-15, 4.1546e-18, 1.6159e-15, 5.1706e-16, 1.3373e-15,\n 9.2503e-15, 2.5899e-15, 6.2976e-16, 1.0946e-15, 1.6639e-15, 1.5069e-15,\n 6.0573e-17, 2.5610e-15, 1.3341e-15, 5.7301e-16, 6.8410e-16, 1.7615e-15,\n 1.5708e-16, 1.5128e-15, 1.1178e-15, 2.3356e-16, 2.0294e-16, 3.6205e-16,\n 2.6389e-15, 4.4771e-17, 2.1936e-15, 2.1627e-16, 2.8119e-16, 6.1039e-16,\n 5.7388e-15, 4.6294e-16, 5.8329e-15, 9.5456e-16, 1.3663e-16, 1.0781e-15,\n 9.7784e-16, 1.0744e-15, 2.3354e-15, 2.9207e-15, 2.4897e-16, 2.7167e-17,\n 2.4257e-16, 3.4305e-16, 5.2606e-16, 1.9671e-15, 6.3809e-16, 7.5313e-15,\n 1.8008e-15, 1.8624e-17, 2.7021e-16, 1.7014e-15, 1.6894e-16, 6.7312e-15,\n 7.1367e-17, 7.6709e-18, 2.2804e-15, 4.9434e-16, 1.6636e-15, 3.7354e-17,\n 2.6880e-15, 1.7648e-15, 1.0851e-15, 6.6898e-16, 1.5897e-15, 2.5638e-17,\n 1.7221e-15, 2.1846e-15, 2.6378e-16, 8.8213e-16, 3.9102e-15, 3.3984e-16,\n 3.2811e-17, 8.0639e-16, 4.7486e-16, 1.9703e-15, 2.3932e-17, 8.7821e-16,\n 6.6086e-16, 1.0828e-16, 4.9010e-15, 4.2807e-15, 2.1565e-15, 5.0385e-16,\n 2.1057e-15, 3.5285e-17, 4.1345e-17, 8.0855e-16, 3.5313e-16, 8.3591e-16,\n 1.2763e-16, 4.2097e-15, 1.3100e-16, 2.9598e-15, 5.0437e-18, 6.4076e-15,\n 4.4747e-15, 2.5411e-15, 6.2619e-17, 1.2502e-15, 2.1528e-15, 6.7026e-16,\n 1.3858e-14, 2.6197e-15, 4.3030e-15, 7.4343e-16, 2.1698e-15, 6.8124e-18,\n 2.2238e-15, 5.0179e-16, 6.5030e-16, 2.8894e-15, 1.3132e-15, 5.0280e-16,\n 7.2466e-17, 2.0266e-16, 1.9510e-15, 1.5151e-16, 1.4214e-15, 1.7762e-15,\n 3.1321e-15, 3.7543e-17, 2.4103e-15, 9.8189e-16, 3.4516e-17, 1.9077e-15,\n 7.0011e-15, 4.0680e-15, 2.5057e-15, 1.7133e-15, 3.6815e-15, 6.1781e-16,\n 1.5069e-15, 2.8161e-15, 4.4966e-15, 5.3138e-16, 1.8642e-15, 2.1003e-15,\n 7.8662e-16, 5.1277e-16, 6.9623e-16, 5.7028e-17, 4.4644e-16, 2.9392e-15,\n 1.1340e-15, 2.7714e-15, 1.5342e-15, 2.2514e-15, 7.7574e-17, 1.7059e-17,\n 5.0343e-15, 5.2809e-15, 8.0437e-17, 7.1257e-15, 2.9000e-15, 1.3472e-15,\n 4.2503e-17, 7.4808e-15, 2.4384e-15, 2.7733e-15, 2.0829e-15, 2.7227e-15,\n 7.5220e-16, 7.2665e-16, 3.8469e-15, 2.4381e-16, 2.5418e-15, 1.9793e-15,\n 1.9114e-15, 5.4206e-16, 9.3537e-16, 1.0070e-15, 9.7294e-16, 2.4986e-16,\n 7.3175e-15, 1.1424e-16, 1.4638e-15, 1.8342e-16, 5.8918e-16, 2.8817e-15,\n 7.4092e-16, 6.5195e-15, 1.8300e-15, 4.2444e-16, 3.0909e-15, 5.1371e-15,\n 1.6833e-15, 5.3638e-18, 1.6066e-17, 5.4277e-15, 6.5256e-16, 1.1812e-15,\n 1.8583e-16, 3.9779e-17, 2.7881e-16, 2.5410e-15, 1.7480e-15, 4.9614e-17,\n 2.4903e-17, 1.5745e-16, 1.1008e-15, 4.2454e-15, 2.3679e-17, 4.7093e-15,\n 1.4284e-16, 2.7467e-15, 4.4352e-16, 1.2524e-15, 1.0101e-15, 3.5429e-16,\n 4.9551e-16, 1.1587e-15, 2.6157e-16, 3.3436e-16, 5.2672e-19, 3.4916e-15,\n 1.3015e-17, 3.8171e-15, 2.6384e-16, 7.1508e-15, 6.2561e-16, 5.4120e-15,\n 2.0550e-15, 1.0278e-15, 6.7234e-15, 1.8915e-15, 1.4353e-15, 2.0351e-15,\n 2.7872e-16, 7.8039e-16, 2.8638e-15, 8.2607e-16, 2.6797e-15, 3.2440e-15,\n 7.7974e-15, 7.6189e-15, 2.4829e-15, 3.1175e-15, 1.2670e-16, 7.0094e-16,\n 4.3173e-16, 3.5554e-17, 8.5172e-16, 3.3274e-17, 8.2712e-16, 1.5187e-15,\n 9.2486e-16, 6.4683e-15, 3.0242e-15, 3.6020e-16, 2.3151e-15, 1.4472e-18,\n 1.3117e-15, 2.0788e-17, 5.0567e-15, 8.5409e-16, 9.0393e-16, 1.7174e-15,\n 6.2405e-17, 3.4920e-15, 4.6917e-16, 2.6311e-16, 9.2596e-15, 6.1826e-16,\n 3.3013e-15, 3.4782e-16, 8.4574e-16, 3.4711e-16, 2.9296e-16, 6.0619e-16,\n 7.8747e-15, 2.5143e-17, 6.4055e-16, 1.4035e-17, 9.8135e-17, 4.1034e-15,\n 3.2740e-16, 7.7285e-15, 8.3001e-16, 4.6076e-15, 2.9690e-16, 8.7450e-17,\n 7.7791e-15, 2.1315e-15, 8.8537e-16, 3.0760e-15, 3.5716e-17, 6.8432e-17,\n 2.4262e-15, 5.7037e-16, 9.7990e-16, 1.3428e-17, 7.0622e-16, 1.7273e-16,\n 1.4686e-15, 2.7521e-15, 1.6554e-15, 2.0518e-15, 6.8411e-16, 4.4083e-17,\n 2.7159e-16, 3.8126e-15, 3.3542e-15, 1.2948e-14, 3.2354e-15, 3.1648e-15,\n 8.1360e-17, 2.2356e-15, 7.1047e-16, 1.8627e-15, 8.7763e-16, 1.1048e-16,\n 4.7878e-16, 3.1657e-15, 2.4561e-15, 7.6818e-16, 4.2431e-15, 1.5425e-15,\n 9.8445e-16, 4.0506e-15, 1.6524e-15, 4.5406e-16, 1.1146e-15, 1.6424e-16,\n 4.1492e-16, 5.8429e-17, 7.0906e-16, 2.2650e-15, 7.9237e-16, 4.8142e-16,\n 7.3402e-16, 3.6627e-15, 1.3125e-15, 4.3596e-16, 1.6992e-16, 2.2789e-15,\n 1.4553e-16, 1.2550e-15, 9.9435e-16, 2.4712e-15, 4.1569e-16, 1.7520e-17,\n 9.9709e-16, 2.4854e-15, 2.2746e-16, 5.1054e-16, 2.0686e-16, 1.0528e-15,\n 1.5190e-15, 1.0722e-17, 6.2359e-16, 7.9920e-16, 3.5105e-16, 3.7157e-17,\n 9.4165e-16, 3.6151e-15, 2.1995e-15, 9.3504e-17, 5.7358e-15, 1.4053e-15,\n 1.5347e-15, 4.6162e-15, 8.7454e-16, 3.4408e-15, 3.3180e-17, 3.5635e-17,\n 7.7786e-18, 3.6360e-15, 1.1903e-17, 7.9735e-17, 1.0945e-15, 1.8619e-17,\n 5.6011e-15, 3.5556e-16, 8.6364e-15, 1.2594e-15, 5.0956e-15, 2.0340e-15,\n 1.5284e-15, 5.4699e-16, 4.1114e-15, 1.5490e-15, 1.8821e-15, 2.0420e-17,\n 4.7803e-15, 4.0836e-16, 1.1719e-15, 1.2162e-15, 2.8383e-15, 3.5205e-15,\n 2.0250e-15, 2.3207e-17, 5.6894e-17, 2.2041e-17, 1.3671e-15, 1.7898e-15,\n 2.3275e-16, 2.0210e-15, 4.9407e-18, 1.2625e-15, 5.0163e-15, 2.1569e-16,\n 1.4077e-16, 5.6244e-16, 4.0011e-17, 2.2195e-15, 2.3668e-15, 1.4281e-15,\n 1.9836e-15, 2.8319e-15, 1.2937e-15, 6.8278e-16, 1.9831e-15, 7.7573e-17,\n 7.7322e-15, 8.0314e-16, 1.9972e-15, 3.3394e-15, 1.1112e-14, 1.5053e-14,\n 2.2228e-16, 2.2584e-15, 4.4340e-16, 1.8956e-15, 1.9864e-15, 1.0441e-16,\n 3.8651e-15, 2.4965e-17, 1.8500e-17, 1.3764e-17, 3.7493e-16, 6.5935e-15,\n 3.1429e-16, 7.6825e-17, 2.2458e-16, 3.4594e-15, 1.4560e-15, 5.3360e-16,\n 5.0418e-17, 5.7327e-16, 1.9815e-15, 1.0836e-14, 5.1484e-15, 6.2283e-15,\n 6.2243e-16, 1.9784e-15, 2.9071e-15, 3.0566e-15, 3.1176e-15, 2.0360e-16,\n 7.2825e-15, 1.3459e-17, 3.0752e-15, 1.9288e-15, 4.8506e-16, 2.4205e-15,\n 3.2494e-16, 6.1592e-15, 5.6129e-16, 1.8581e-15, 1.5671e-15, 1.0168e-14,\n 4.9028e-17, 1.7619e-15, 4.7141e-16, 1.7926e-15, 3.2752e-18, 5.1215e-16,\n 1.5143e-17, 8.2568e-16, 9.4122e-18, 1.4646e-15, 4.9096e-16, 1.5852e-16,\n 4.2956e-15, 3.8726e-17, 6.1288e-15, 2.1854e-15, 2.6649e-15, 2.0314e-15,\n 5.3096e-16, 3.0260e-15, 6.8855e-16, 1.5001e-15, 2.5788e-15, 5.1664e-16,\n 1.1420e-15, 6.9522e-16, 5.8819e-15, 1.4061e-15, 1.2840e-16, 3.0349e-17,\n 1.6207e-17, 4.2385e-15, 3.6519e-15, 1.4338e-16, 1.5749e-15, 1.6981e-15,\n 2.6643e-16, 1.5305e-15, 7.1201e-15, 1.6568e-15, 7.1840e-16, 9.2695e-16,\n 2.8526e-17, 8.1304e-16, 4.2347e-15, 3.0123e-15, 1.8465e-16, 2.9445e-15,\n 3.3333e-15, 2.3112e-17, 1.9178e-15, 3.1982e-15, 1.5118e-15, 5.2831e-16,\n 7.4066e-15, 8.0914e-16, 5.3977e-15, 2.4419e-15, 1.9086e-15, 4.9071e-16,\n 2.1564e-16, 5.5463e-17, 6.5837e-18, 1.9953e-15, 1.4357e-15, 1.3613e-15,\n 1.1749e-15, 2.1719e-15, 2.7924e-16, 7.5627e-16, 1.4185e-14, 8.7244e-16,\n 3.1817e-15, 3.7518e-16, 1.0415e-15, 6.4170e-17, 1.5496e-15, 3.7346e-15,\n 2.3740e-16, 3.7575e-17, 9.1797e-16, 6.9023e-16, 1.2087e-15, 1.2826e-15,\n 4.7319e-15, 2.2440e-15, 7.4951e-16, 8.7778e-16, 6.3252e-17, 2.5307e-15,\n 6.9197e-16, 2.9538e-15, 3.3398e-15, 3.5278e-15, 1.2288e-14, 3.4823e-16,\n 3.5750e-15, 3.2119e-16, 3.1612e-16, 2.3346e-15, 3.1206e-15, 7.1103e-17,\n 6.6830e-16, 9.2062e-17, 4.0026e-15, 5.0366e-17], device='cuda:0')" + }, + "46": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, 2.1433e-17, -4.6550e-17, ..., 4.1283e-16,\n 6.7780e-16, 6.4478e-17],\n [ 5.6052e-45, 5.0882e-16, 4.8856e-17, ..., 2.8510e-15,\n -3.8511e-15, 5.8492e-17],\n [-5.6052e-45, -3.7132e-16, -1.9358e-18, ..., 3.1399e-16,\n 3.4015e-15, 5.5993e-17],\n ...,\n [ 5.6052e-45, 8.2374e-17, -1.7122e-17, ..., 1.8483e-17,\n -1.2426e-16, 7.7792e-18],\n [ 5.6052e-45, 1.0829e-15, -7.2059e-18, ..., -2.4201e-15,\n -5.4774e-15, -2.2078e-17],\n [ 5.6052e-45, 1.5552e-16, -1.8789e-18, ..., 7.0287e-17,\n -2.3469e-16, 4.5953e-18]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.5327e-16, 7.5361e-16, 5.4977e-16, ..., 9.8667e-18, 3.0027e-17,\n 4.2872e-16],\n [4.5106e-15, 7.2513e-15, 9.1953e-15, ..., 6.6239e-18, 4.1431e-16,\n 7.7448e-15],\n [5.9444e-16, 1.9127e-15, 3.3590e-15, ..., 1.1628e-18, 6.7663e-17,\n 1.4810e-15],\n ...,\n [3.6574e-17, 8.7140e-17, 7.5441e-17, ..., 1.0404e-18, 1.0974e-17,\n 5.1549e-17],\n [2.4160e-16, 1.8393e-15, 3.6257e-16, ..., 6.5226e-18, 8.0654e-17,\n 1.9755e-16],\n [1.1935e-16, 3.8701e-16, 5.3638e-16, ..., 3.1685e-19, 1.2703e-17,\n 3.1474e-16]], device='cuda:0')" + }, + "47": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-1.8161e-12, -6.0056e-13, 1.2737e-12, 2.6444e-12, 7.8172e-14,\n -3.9007e-13, -5.0724e-12, -9.8705e-13, -9.1957e-13, -1.8464e-12,\n 1.2286e-11, -4.5260e-12, -3.4268e-13, -9.6514e-13, 4.8554e-12,\n 1.8992e-12, -1.6440e-12, -8.4024e-13, 2.9937e-12, -5.9314e-13,\n -8.0666e-13, 3.0713e-12, -3.5505e-12, -4.5527e-13, -9.0109e-13,\n -1.1100e-12, 1.0134e-12, 1.1288e-12, 1.3350e-12, 7.5544e-13,\n -7.7002e-13, 5.4605e-12, 1.3875e-12, -1.4750e-12, 3.2993e-12,\n -6.0872e-12, 1.3501e-12, -1.1669e-12, 2.5319e-12, -1.0026e-12,\n -4.1431e-12, -4.0615e-13, 2.0888e-13, -6.5185e-12, 2.9093e-12,\n 2.1958e-12, -2.6132e-12, -1.7540e-12, -9.3055e-13, 7.9377e-13,\n 9.0311e-13, 1.8764e-12, -1.5510e-12, -1.4434e-12, -1.1849e-12,\n 2.2589e-12, 1.2709e-12, -1.5263e-12, -1.6934e-12, 4.8400e-12,\n -5.8038e-12, -1.1693e-12, 3.1967e-12, 3.0429e-13, 2.5772e-12,\n -1.2411e-12, -7.0252e-12, 2.0924e-12, 1.5319e-12, -1.4169e-12,\n 4.7948e-12, -1.3508e-12, -1.2474e-12, -3.1585e-12, -1.2294e-12,\n -6.9630e-12, -1.2332e-13, -6.0539e-13, -2.0025e-12, -4.2476e-12,\n 2.3729e-12, -2.2638e-13, 7.0505e-12, 4.8469e-12, -1.2414e-12,\n -1.3516e-12, 4.1542e-12, -1.2847e-12, -1.2680e-12, -2.2996e-12,\n -6.6683e-13, 3.1077e-12, -4.3862e-12, -4.5309e-12, -1.0926e-12,\n -1.0357e-12, -3.7533e-12, -8.9387e-14, -2.6206e-12, -1.1213e-12,\n -5.4179e-12, -3.3707e-13, 2.6104e-12, 8.3170e-13, 3.0743e-12,\n -2.4178e-12, 8.4296e-13, -1.2541e-12, -1.1331e-12, -1.1993e-12,\n -3.9147e-12, -1.1928e-12, -4.2935e-13, -1.1523e-12, -1.2676e-12,\n -1.0128e-12, -2.2868e-12, 2.7856e-12, -2.8670e-12, 1.1994e-12,\n 1.9337e-12, -4.7551e-12, -8.5631e-13, -9.0743e-13, 6.4476e-12,\n -2.9037e-12, -1.1668e-12, -3.4776e-13, -3.6587e-12, 1.5339e-12,\n -4.3883e-12, -1.3915e-12, -2.7570e-13, -4.5961e-12, -1.2458e-12,\n 1.9291e-13, -3.5668e-13, -6.1913e-12, 2.0368e-12, -1.4457e-12,\n -7.6911e-13, -9.3653e-13, -1.0196e-12, -2.0575e-12, 5.5317e-13,\n 7.4360e-12, 1.5871e-12, -1.4437e-12, -7.3578e-13, -3.3499e-12,\n -1.6948e-14, 7.0621e-12, -1.5880e-12, -1.4895e-13, 5.4870e-12,\n -9.8718e-13, 3.7313e-14, 7.7076e-13, -6.2517e-12, 7.2181e-13,\n -1.5726e-12, -1.1442e-12, -6.6045e-13, 1.6373e-13, -9.7388e-13,\n 1.4777e-12, -9.4496e-13, -1.7389e-12, 2.3240e-12, 9.6313e-13,\n -1.6850e-12, 8.8799e-12, -1.7261e-12, -1.0466e-12, 1.4259e-11,\n -4.0203e-12, 8.4883e-12, -1.2844e-12, -1.2293e-12, -6.1940e-13,\n 4.5491e-12, 5.4187e-12, 3.4790e-13, 7.2783e-12, -3.7509e-12,\n 3.8595e-12, 2.7761e-12, -2.4897e-12, -2.3355e-12, -7.6327e-12,\n 2.7273e-15, 3.2120e-12, 8.1283e-13, -3.6034e-12, 2.2675e-12,\n -2.7698e-12, 2.1535e-12, -2.1533e-12, -2.8045e-12, -5.1321e-13,\n 6.3879e-12, -5.6863e-13, 1.3514e-12, -2.7484e-13, -3.7551e-12,\n -1.3783e-11, -9.3756e-13, 5.2863e-13, -4.3292e-13, 5.2942e-13,\n 5.5082e-12, 3.8064e-12, -7.4181e-13, 9.2682e-13, -6.6331e-13,\n 1.2283e-11, -1.2116e-12, 8.8549e-12, -1.2211e-12, 2.2606e-12,\n -2.5192e-12, 6.9965e-12, 4.0391e-12, 2.1497e-12, -2.7955e-12,\n -1.1639e-12, 2.0772e-12, -3.9090e-13, 8.7257e-12, 2.3241e-13,\n -1.8799e-12, 5.7720e-13, -2.2741e-12, 1.1105e-11, -3.2900e-12,\n -2.9302e-12, -5.9794e-12, -4.3816e-12, -7.3350e-13, 3.4164e-12,\n 8.3348e-12, -3.0697e-12, 1.4368e-13, -5.8345e-12, -5.5734e-13,\n -2.3597e-12, -4.8796e-13, -1.2524e-12, -2.1667e-13, 4.8273e-12,\n -1.1988e-12, -6.3879e-12, 2.7394e-12, -1.2502e-12, -1.5703e-12,\n 1.7323e-12, -5.3047e-13, -1.0705e-12, -1.6100e-12, 1.9675e-12,\n -1.7418e-12, 7.7760e-13, 3.6665e-12, 3.3248e-12, -1.2814e-12,\n -1.5491e-12, -4.4319e-12, 4.0754e-12, 3.5510e-12, -3.5778e-13,\n -2.0658e-12, -6.4083e-13, -9.2258e-12, -1.2874e-12, -1.6924e-12,\n -5.5149e-13, -1.2833e-12, -2.4183e-13, -4.7858e-12, -7.9251e-13,\n -4.1534e-12, 1.0851e-11, 5.9688e-13, 1.0415e-14, -1.2525e-12,\n -9.4420e-13, -1.6768e-12, -1.0734e-12, -1.0658e-12, 3.4977e-12,\n -1.1167e-12, -3.5304e-12, -9.9076e-13, -1.1699e-12, 3.5820e-14,\n -6.8216e-12, -9.3610e-13, -3.0057e-12, 3.1693e-12, -1.1365e-12,\n -1.0415e-11, -4.5691e-12, -2.4363e-12, -1.3276e-12, -1.0963e-12,\n -9.4890e-13, -3.9099e-12, -9.4307e-12, 5.9827e-12, 2.0525e-12,\n -5.8526e-12, -1.3403e-12, -4.4708e-12, -2.4754e-13, 1.0350e-12,\n 4.1316e-12, -2.7922e-12, 3.6293e-12, -5.5081e-12, 2.7367e-14,\n -6.7163e-13, 6.5314e-13, -2.0277e-12, 4.6213e-13, -1.0059e-12,\n -3.4392e-12, 1.3169e-13, 1.0103e-11, -6.0679e-12, 6.1934e-12,\n 4.7349e-12, 6.9598e-12, -1.5557e-12, 1.5995e-12, 1.9099e-11,\n -1.3278e-12, 2.5133e-12, -1.1751e-12, 1.1517e-11, 4.0645e-13,\n -1.7224e-13, 2.1427e-12, 3.4898e-12, -6.0005e-12, -2.3201e-12,\n -4.2744e-12, 8.6096e-13, 5.3329e-12, 2.0508e-12, 3.4023e-12,\n -8.3529e-13, 7.4039e-12, -1.8521e-14, -1.5937e-12, 4.0523e-13,\n -8.4469e-13, -1.1078e-13, 6.7141e-13, 8.6718e-13, -2.1092e-12,\n 4.7770e-12, 2.4736e-12, -1.2754e-12, -2.2224e-12, 2.9133e-12,\n 5.8994e-12, 4.8149e-12, -1.3596e-12, -7.0020e-13, -2.8987e-12,\n -9.1395e-13, 7.6165e-12, -8.4506e-13, -9.1706e-13, -2.8745e-12,\n 4.8219e-12, -1.4046e-12, 1.4186e-11, -1.7775e-12, -2.7131e-12,\n 2.0478e-12, -2.0391e-12, 3.5215e-12, -2.5644e-12, 1.9898e-12,\n 7.6149e-13, 3.3371e-12, -1.3619e-12, -4.5639e-13, 1.0489e-11,\n 3.8028e-12, -2.3346e-12, -3.7633e-12, -1.2930e-12, 2.3064e-13,\n 1.3835e-11, -5.5761e-13, -1.8181e-12, -9.2965e-13, 7.9659e-12,\n -5.7241e-13, -5.1063e-13, -2.2191e-12, 2.7890e-12, 7.1489e-12,\n 3.2418e-13, -1.6667e-12, -8.8775e-13, -3.3799e-12, -8.2576e-13,\n 3.8241e-13, -1.2840e-12, -1.6914e-12, -2.9789e-12, -8.0685e-12,\n -1.8088e-12, 8.5685e-12, 4.3492e-12, -6.4295e-13, 2.9920e-12,\n -8.9605e-13, 4.3990e-12, -3.3109e-12, -7.2557e-13, 2.3785e-12,\n 1.2650e-12, 2.1695e-12, -5.8194e-14, -2.0004e-12, -3.7777e-12,\n -2.3520e-12, -9.1970e-14, -8.4598e-13, -1.5760e-12, 6.9338e-13,\n 1.1572e-12, -3.5343e-12, 2.2061e-12, 3.0369e-12, -3.4368e-12,\n 2.3320e-12, 2.8010e-13, -1.9390e-12, -2.4589e-12, 2.1799e-12,\n -6.6127e-13, -8.5843e-12, 3.1590e-12, -1.3641e-12, -1.1352e-12,\n 4.4353e-12, 4.5670e-12, 1.8365e-12, -1.1664e-12, -8.4755e-13,\n 1.8915e-12, 2.7238e-13, -1.3420e-12, -1.2228e-12, -1.4121e-12,\n -6.2638e-13, 3.2951e-13, 1.1641e-12, -8.1785e-13, 5.9754e-12,\n -6.7213e-12, -5.6890e-12, -8.3408e-13, -1.9843e-12, -3.2242e-12,\n -1.1146e-12, 5.1535e-13, -2.1897e-12, -1.4604e-12, 2.0857e-12,\n -1.4019e-12, -5.1264e-12, -1.7214e-12, -1.2168e-12, -9.8873e-13,\n 3.3395e-12, -9.2098e-12, -3.8224e-13, 3.4566e-12, 8.5811e-12,\n -4.2101e-12, 6.7089e-12, -7.4547e-13, 8.2858e-12, 2.7963e-12,\n -5.7893e-13, 8.1841e-12, -3.8880e-12, 6.3884e-12, 3.9499e-12,\n -3.4190e-12, 3.3378e-12, -8.1544e-13, 2.5833e-12, -4.3286e-12,\n -2.2539e-12, -9.5701e-13, 1.1619e-12, -2.2345e-12, -1.8572e-12,\n 1.5176e-12, -1.1437e-12, -4.3063e-12, 3.2442e-13, -1.2965e-12,\n -4.1502e-12, 2.3660e-12, -2.4904e-13, -3.4786e-12, -4.2221e-12,\n -1.3061e-12, -6.5070e-13, -5.3459e-12, -1.4522e-12, 4.0398e-12,\n -1.5043e-12, 3.0292e-14, -5.4598e-12, 4.1679e-12, -5.0322e-13,\n -4.4693e-12, 1.6679e-13, -1.0745e-12, 6.5887e-13, -1.3151e-12,\n -1.1480e-12, 4.9724e-12, -6.4427e-12, 1.5097e-12, -4.2330e-13,\n -8.2673e-13, 2.7576e-12, 3.5202e-12, -2.4736e-12, 6.9572e-13,\n -1.0725e-12, 3.3573e-12, -4.2093e-12, 8.2614e-12, -1.2892e-12,\n 4.8746e-13, 3.0892e-12, -6.3000e-13, 2.1980e-12, -1.8838e-12,\n -7.3154e-13, -1.8208e-12, -7.3191e-13, -7.2911e-12, -6.7571e-12,\n 1.9758e-12, 8.4857e-13, -9.2060e-13, -1.9358e-12, -1.1581e-12,\n 7.3345e-13, 2.6964e-12, 4.8375e-12, -3.1673e-12, -4.1345e-12,\n 6.9574e-14, -5.4544e-13, -2.6414e-13, 1.9118e-12, -6.7279e-14,\n 7.3113e-12, 2.7867e-12, 5.1522e-12, 3.5501e-12, -5.4652e-12,\n -1.3579e-12, -1.2472e-12, -1.2140e-12, 3.2363e-12, 1.9905e-13,\n -1.4198e-12, 1.5234e-13, -1.3151e-13, 4.1856e-14, -3.8506e-13,\n -1.1774e-12, -1.5369e-12, 2.0232e-12, 4.1354e-13, -7.2709e-13,\n 4.0370e-13, -1.2030e-12, -1.7658e-12, -2.6969e-12, 6.1730e-13,\n -4.9837e-13, -2.8146e-13, 4.4264e-13, -1.0392e-12, -1.2278e-12,\n -1.8742e-12, -1.1069e-12, -1.2914e-12, 1.5080e-12, 1.0109e-12,\n -1.1360e-12, -6.9702e-12, 2.6283e-12, 2.0635e-12, 2.5810e-13,\n 4.6708e-12, -2.1390e-12, -1.2915e-12, -5.8021e-13, -1.5484e-12,\n -2.1696e-12, -2.5957e-13, -3.2487e-12, -1.3457e-12, -1.1489e-12,\n -3.9330e-12, 3.9902e-12, 7.9867e-12, 9.3116e-12, -2.2263e-12,\n -3.4896e-12, 4.6033e-12, -1.4529e-12, 4.5516e-13, 7.6132e-12,\n -1.0127e-12, 4.1427e-12, -3.2475e-12, 6.9180e-12, -5.7959e-13,\n 1.2155e-12, -1.0090e-12, -2.9241e-13, -1.0436e-11, -1.0947e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.2551e-12, 5.8880e-12, 1.3849e-12, 2.8188e-12, 2.4701e-13, 9.1538e-13,\n 1.1648e-12, 5.2536e-13, 1.2593e-13, 2.2747e-12, 8.6164e-12, 4.0568e-13,\n 4.2162e-13, 1.9880e-12, 4.2352e-13, 1.9183e-12, 1.4381e-12, 2.2292e-12,\n 7.7317e-13, 2.8894e-12, 2.1435e-13, 1.3031e-13, 1.2160e-12, 3.2051e-13,\n 5.4340e-13, 1.1399e-12, 1.4201e-13, 3.4329e-13, 5.5222e-13, 6.0365e-13,\n 2.2372e-12, 8.2928e-12, 5.1866e-12, 6.7099e-14, 4.6173e-13, 4.3948e-13,\n 4.1119e-12, 1.7341e-13, 2.4482e-13, 5.9229e-14, 5.8936e-14, 2.6212e-12,\n 1.2467e-12, 3.0214e-12, 1.1127e-12, 2.5422e-12, 1.9490e-11, 3.1962e-12,\n 1.1721e-13, 3.6795e-13, 5.6849e-12, 8.6456e-14, 1.6926e-12, 3.7620e-12,\n 6.7753e-12, 4.2508e-12, 1.7604e-13, 1.2061e-13, 1.9841e-12, 1.5176e-12,\n 2.7973e-12, 7.9790e-13, 1.4585e-12, 8.4748e-12, 2.6273e-13, 8.5587e-14,\n 2.8116e-12, 2.2946e-12, 1.4513e-13, 6.2329e-14, 2.0056e-12, 5.8371e-13,\n 2.0005e-13, 2.7151e-12, 3.1359e-12, 4.2989e-12, 1.3127e-12, 7.6033e-13,\n 1.8415e-12, 8.4583e-12, 2.7678e-13, 1.1922e-12, 4.1129e-12, 3.4410e-13,\n 2.9429e-12, 5.5253e-13, 5.9283e-12, 3.5328e-12, 1.6620e-12, 2.0311e-12,\n 1.7397e-13, 1.4198e-12, 3.8834e-14, 2.8069e-12, 2.9989e-13, 9.1189e-13,\n 1.9845e-13, 2.8953e-13, 8.0922e-13, 1.2465e-12, 2.0496e-12, 1.9922e-12,\n 8.0428e-13, 8.2174e-13, 1.9479e-12, 6.1211e-13, 1.5138e-13, 1.3282e-12,\n 1.4461e-12, 6.5768e-14, 1.2504e-12, 2.8248e-13, 5.6286e-13, 3.8367e-14,\n 5.2451e-13, 7.1449e-12, 2.8379e-12, 5.5292e-13, 3.4893e-13, 4.4921e-12,\n 9.3212e-12, 2.3739e-12, 1.7277e-13, 1.0105e-12, 1.3201e-12, 6.1756e-12,\n 9.8004e-13, 8.7907e-14, 2.6366e-12, 1.7462e-12, 1.4755e-12, 1.2685e-12,\n 1.4809e-13, 2.2085e-13, 8.7190e-13, 1.3390e-12, 6.3431e-12, 9.8737e-13,\n 1.7183e-12, 7.2913e-13, 2.8476e-12, 5.4961e-13, 4.5076e-12, 3.3247e-13,\n 4.2032e-13, 6.8922e-12, 2.7077e-12, 9.3132e-12, 1.6290e-13, 2.9985e-14,\n 4.6514e-13, 7.3989e-12, 7.0967e-14, 4.0046e-13, 2.3438e-12, 1.6234e-12,\n 1.5458e-12, 5.5978e-12, 4.2948e-12, 2.3148e-12, 1.5449e-13, 6.3206e-14,\n 7.9991e-13, 2.7937e-13, 3.7549e-12, 8.1880e-13, 1.7934e-13, 4.7989e-12,\n 1.0323e-11, 2.8975e-12, 8.0082e-14, 4.8818e-12, 4.4653e-12, 5.6008e-12,\n 2.6446e-11, 3.4082e-12, 4.1079e-12, 2.4314e-12, 5.3961e-13, 6.7862e-13,\n 4.1545e-13, 2.0085e-12, 2.2827e-13, 6.7200e-12, 6.3864e-13, 1.3737e-11,\n 3.5781e-12, 1.9931e-12, 7.1914e-13, 1.9457e-13, 7.7719e-14, 1.0967e-11,\n 9.6731e-13, 1.9348e-13, 2.0232e-13, 2.1850e-12, 3.4091e-12, 4.3627e-12,\n 8.6264e-14, 4.3742e-12, 3.9034e-13, 9.0359e-13, 8.2562e-13, 1.0273e-13,\n 7.4427e-13, 9.1377e-12, 5.3186e-13, 3.9194e-13, 1.5184e-12, 4.5133e-14,\n 2.7534e-12, 5.2938e-12, 4.9317e-13, 2.1937e-13, 6.7384e-13, 5.9598e-12,\n 8.0943e-13, 1.6438e-12, 1.7696e-13, 6.0152e-12, 8.7682e-13, 4.5700e-12,\n 5.5504e-12, 2.6725e-13, 9.8965e-13, 6.4678e-14, 2.2519e-12, 1.3689e-13,\n 4.5302e-12, 2.0438e-12, 1.8472e-12, 5.8325e-12, 3.0435e-12, 6.6223e-12,\n 4.4564e-13, 1.3889e-13, 6.3527e-12, 1.7989e-12, 4.9223e-12, 2.2273e-12,\n 1.7335e-12, 3.6645e-13, 7.7797e-12, 1.3520e-12, 6.1443e-13, 2.2263e-14,\n 2.7546e-13, 1.5141e-13, 1.6741e-12, 7.3700e-12, 3.5161e-13, 1.3512e-12,\n 3.3743e-12, 1.9859e-13, 3.3370e-12, 1.4068e-12, 2.8647e-12, 4.4770e-13,\n 2.0448e-12, 3.5023e-13, 2.3820e-13, 8.8028e-13, 2.0858e-12, 3.9809e-12,\n 5.3495e-13, 2.8372e-12, 4.9908e-13, 5.8470e-12, 1.9328e-12, 1.4578e-13,\n 2.0450e-12, 1.6844e-13, 5.0132e-12, 3.8502e-12, 3.6400e-12, 4.4254e-12,\n 2.3097e-12, 1.6380e-12, 4.6012e-13, 1.2138e-13, 2.0695e-12, 9.7608e-12,\n 4.9722e-12, 6.7860e-13, 4.7755e-12, 6.9782e-13, 1.5786e-12, 1.4413e-12,\n 6.3718e-14, 2.3859e-12, 1.3106e-12, 1.3128e-11, 3.3360e-13, 9.1376e-13,\n 1.1387e-13, 4.3875e-12, 4.2705e-12, 3.7700e-12, 4.5466e-14, 1.5559e-13,\n 3.8799e-12, 2.3112e-12, 1.4161e-12, 3.2096e-12, 1.5894e-12, 7.1349e-13,\n 1.9738e-11, 7.6051e-12, 3.3999e-12, 1.0248e-13, 1.4185e-12, 7.4589e-13,\n 1.5684e-12, 3.0146e-12, 3.2316e-14, 7.0195e-12, 4.3724e-13, 1.6217e-12,\n 3.5828e-12, 3.2986e-12, 3.0121e-13, 2.1557e-13, 8.8746e-14, 4.7482e-13,\n 1.0873e-12, 3.3061e-12, 8.4184e-13, 4.6568e-12, 6.9046e-12, 3.9023e-12,\n 7.3861e-12, 2.6724e-12, 1.0017e-13, 2.4392e-13, 1.4300e-11, 4.3244e-13,\n 7.7789e-12, 9.8109e-13, 4.6495e-12, 1.5556e-13, 5.3981e-13, 9.9636e-13,\n 8.0655e-12, 1.0725e-11, 1.1743e-12, 1.1750e-12, 1.9407e-12, 7.0930e-12,\n 2.6257e-13, 3.6767e-12, 2.1419e-12, 4.1095e-12, 4.0388e-12, 2.0376e-12,\n 1.4757e-11, 2.3442e-12, 2.0680e-13, 1.5899e-13, 1.7345e-13, 2.9536e-13,\n 4.7303e-12, 1.3937e-12, 2.6748e-12, 7.3682e-13, 4.2158e-13, 8.7516e-13,\n 1.5389e-12, 7.8220e-14, 1.2242e-13, 1.1370e-13, 9.6008e-13, 3.2918e-12,\n 1.8809e-13, 1.5191e-12, 3.6723e-12, 1.1235e-11, 1.9127e-12, 8.8190e-12,\n 3.7613e-12, 2.5408e-12, 5.5230e-13, 1.5240e-12, 4.2762e-12, 6.1349e-14,\n 3.9501e-12, 1.4208e-11, 3.9288e-12, 3.3894e-12, 2.7111e-12, 8.1672e-12,\n 2.1918e-12, 1.1069e-12, 2.1216e-12, 1.1788e-12, 3.0333e-13, 4.3406e-12,\n 2.4647e-13, 1.0734e-11, 7.0251e-14, 2.1511e-12, 4.1342e-12, 1.4593e-12,\n 1.1392e-13, 3.5489e-12, 1.9180e-13, 1.0051e-12, 4.7434e-13, 1.8193e-12,\n 1.0674e-12, 6.2614e-14, 1.0023e-13, 1.6733e-13, 4.2014e-13, 1.1464e-11,\n 2.1618e-12, 4.9481e-13, 2.7781e-12, 5.2373e-12, 7.5321e-13, 2.5900e-12,\n 1.1900e-12, 3.6107e-12, 7.6080e-13, 5.9591e-13, 1.6638e-12, 2.3898e-12,\n 2.3635e-12, 1.5301e-12, 5.3152e-14, 1.2207e-11, 1.8984e-12, 6.7018e-12,\n 7.6994e-13, 8.6422e-12, 1.1186e-12, 6.0563e-12, 4.9779e-12, 2.5984e-12,\n 2.6418e-12, 1.8480e-13, 3.7112e-12, 3.2939e-12, 9.4671e-12, 2.3010e-12,\n 2.1898e-12, 8.6052e-14, 1.3689e-11, 5.6137e-12, 1.7822e-13, 1.3156e-13,\n 2.4307e-13, 7.2451e-13, 1.5045e-12, 1.1858e-13, 6.1087e-13, 4.2637e-12,\n 2.8302e-13, 2.1202e-12, 5.7960e-13, 8.9034e-14, 3.6982e-13, 8.8420e-12,\n 2.5225e-13, 2.6067e-12, 6.5937e-12, 1.2464e-11, 2.9241e-12, 2.1925e-12,\n 6.2696e-14, 5.3437e-13, 1.3067e-13, 1.0419e-13, 7.6180e-12, 6.4502e-13,\n 8.1851e-12, 7.3617e-13, 1.4294e-12, 2.1159e-12, 1.6795e-12, 1.8669e-12,\n 2.4211e-12, 1.2131e-11, 1.7952e-13, 1.7145e-13, 1.2415e-12, 3.9363e-12,\n 4.7154e-12, 3.6176e-12, 9.9559e-13, 7.2789e-12, 2.8262e-13, 5.3430e-12,\n 1.9169e-13, 5.2977e-12, 2.8838e-12, 1.2107e-12, 1.6776e-13, 3.3136e-14,\n 1.6671e-13, 4.0527e-12, 5.5549e-13, 3.6758e-12, 2.9032e-12, 7.0290e-12,\n 9.6808e-14, 1.1863e-12, 7.5120e-13, 1.5242e-12, 4.6666e-13, 2.7948e-13,\n 4.6171e-12, 1.6886e-12, 3.2139e-12, 1.3649e-12, 1.1040e-12, 1.1374e-13,\n 2.3670e-12, 4.7069e-12, 3.3887e-13, 1.8972e-13, 6.3812e-12, 1.6400e-13,\n 9.9384e-12, 2.2083e-12, 1.9276e-13, 6.6737e-12, 9.8125e-13, 9.0314e-12,\n 3.5943e-12, 1.1983e-11, 7.8036e-13, 1.3469e-13, 1.0985e-11, 1.3457e-11,\n 5.4092e-12, 1.9463e-14, 2.4909e-12, 6.2303e-12, 4.0968e-12, 2.4622e-12,\n 6.8007e-13, 2.2243e-12, 1.6807e-12, 3.0151e-12, 4.7489e-12, 2.7582e-12,\n 4.4729e-13, 1.1789e-12, 1.6120e-13, 2.8492e-12, 2.7956e-12, 1.2068e-12,\n 4.0870e-13, 6.1119e-12, 4.3098e-12, 1.1513e-12, 1.8387e-13, 9.4427e-14,\n 8.1143e-13, 9.0238e-14, 1.4522e-11, 2.3047e-13, 3.6807e-12, 2.3841e-12,\n 1.7101e-12, 7.6279e-13, 6.7063e-13, 6.8688e-13, 5.3356e-12, 3.1788e-13,\n 1.3859e-12, 2.9724e-12, 1.7509e-13, 1.3010e-12, 4.9428e-12, 3.4872e-13,\n 8.8024e-14, 7.0547e-13, 4.3205e-12, 1.1178e-13, 4.1419e-13, 2.6282e-12,\n 5.1192e-13, 2.9983e-12, 1.3815e-12, 4.8240e-13, 2.9344e-13, 1.9683e-12,\n 2.3467e-13, 1.5612e-12, 1.9548e-13, 4.4235e-13, 1.3075e-12, 2.1367e-12,\n 1.4330e-12, 3.8436e-12, 2.3115e-12, 3.8184e-13, 1.4504e-12, 1.8942e-12,\n 4.6338e-13, 5.0087e-14, 5.9041e-13, 2.6353e-12, 3.2926e-11, 2.3390e-12,\n 1.1643e-12, 2.5672e-12, 1.9481e-12, 3.6842e-13, 5.4397e-12, 3.3105e-12,\n 5.1603e-12, 7.0356e-13, 1.8371e-13, 1.8045e-12, 6.3530e-12, 2.6107e-12,\n 1.3816e-12, 1.0962e-12, 2.1612e-12, 9.4572e-13, 2.8679e-12, 1.5965e-11,\n 3.4261e-13, 2.5519e-12, 9.9341e-12, 3.0016e-14, 9.6859e-13, 2.1603e-12,\n 6.8559e-14, 1.6989e-12, 5.0076e-13, 1.9254e-12, 3.7142e-12, 2.2182e-13,\n 8.1339e-14, 1.4506e-13, 2.7402e-12, 2.1995e-13], device='cuda:0')" + }, + "48": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.8234e-14, 1.6493e-13, -3.4619e-14, 7.1244e-14, 7.6069e-15,\n 2.0244e-14, 1.1649e-13, 2.2162e-14, -5.8695e-15, 1.6146e-14,\n 1.6540e-13, 9.3328e-15, 3.5326e-14, 1.7681e-13, 1.1707e-13,\n 1.6040e-14, 5.6080e-14, -1.8091e-16, 1.2920e-14, 1.5877e-13,\n 1.4588e-14, 1.0081e-14, 2.8608e-14, -6.3247e-15, 1.1206e-15,\n 3.2596e-14, 8.9205e-15, -4.7797e-15, 1.8415e-14, 1.1673e-14,\n -1.1493e-14, 8.7433e-14, 7.5801e-14, 1.8749e-14, 1.9394e-13,\n 8.5926e-15, -1.2154e-14, -1.9069e-15, 1.0751e-13, 1.8796e-14,\n 3.5651e-14, 7.4532e-15, 2.6872e-13, -1.9145e-14, 1.2705e-13,\n 4.6427e-16, 8.1026e-14, 4.7643e-14, 3.0471e-14, 1.5462e-15,\n -1.0519e-14, 1.0866e-14, 3.5390e-14, -5.6595e-15, 3.0254e-15,\n 1.5086e-14, -1.2794e-14, 2.4243e-14, 7.6145e-15, 8.3526e-14,\n -2.2285e-14, 1.6914e-14, 2.9647e-13, 3.7303e-15, 2.8779e-13,\n 8.6361e-16, -3.6052e-14, 1.5923e-13, -2.1866e-15, 1.2126e-15,\n 2.5544e-13, 1.6878e-14, 2.3275e-14, 1.5496e-14, -5.0796e-15,\n -1.4820e-14, 1.9369e-14, 1.0192e-14, 3.3857e-14, 1.3344e-14,\n 4.3495e-15, -2.0139e-14, 1.3533e-13, 1.8726e-13, -4.0746e-15,\n 5.4129e-15, 3.1527e-13, 2.3437e-15, -3.8039e-16, 3.1159e-14,\n 3.0900e-14, 2.4521e-13, 5.0763e-15, -1.4183e-14, -8.4842e-16,\n 2.9245e-14, -1.7153e-15, -1.4690e-14, 2.1085e-15, -1.0908e-14,\n -2.5089e-14, 6.4300e-15, 2.2579e-14, -4.8335e-15, 1.0765e-13,\n 3.2148e-15, 2.8884e-14, 2.7636e-14, -1.9031e-14, -4.9840e-15,\n -4.6649e-15, 3.0498e-16, 2.5684e-13, -1.3915e-15, -1.3933e-15,\n -6.4558e-15, 4.4141e-14, 1.5183e-15, -6.2096e-17, 6.3116e-15,\n 1.5693e-13, -1.1659e-14, 1.7414e-14, 1.9677e-14, 1.2303e-13,\n 3.1160e-13, 2.5998e-15, 6.0058e-15, 4.8671e-15, 1.4484e-13,\n 1.3439e-13, -1.7174e-16, 1.9740e-14, 1.1382e-14, 3.8471e-14,\n -2.8535e-14, -6.2292e-15, -3.6080e-14, -1.4571e-14, 2.4744e-15,\n -1.2050e-14, -3.2550e-15, -1.0321e-15, 4.7809e-14, 3.3501e-14,\n 1.5063e-13, 2.7279e-14, 6.8182e-14, 4.7158e-14, -3.5273e-16,\n -1.1537e-14, 2.0371e-13, 1.2833e-14, 1.7276e-14, 8.9994e-14,\n -6.6975e-16, 3.9314e-15, 3.3874e-14, 4.1636e-14, 2.4635e-13,\n 3.4661e-14, -2.7673e-15, 9.6747e-15, -7.9718e-15, 4.5377e-14,\n -1.2398e-14, 1.4631e-14, -8.8967e-15, 8.6542e-15, 7.0808e-15,\n 2.4797e-14, 4.6433e-13, 1.8988e-15, -4.9881e-15, 3.7284e-13,\n -9.2118e-15, 1.4335e-13, 4.6416e-14, 1.8561e-15, 1.9018e-14,\n 2.4226e-16, 1.2321e-13, 9.5230e-15, 5.5227e-13, -3.0343e-16,\n 3.2900e-13, 1.8478e-13, 5.8078e-14, 6.4723e-15, -8.3770e-14,\n 1.8030e-14, 9.4285e-14, 2.3186e-14, 8.5536e-16, -2.2173e-14,\n 1.4515e-14, 1.7960e-14, 1.4035e-15, 4.1831e-15, 1.5898e-13,\n 2.5273e-13, -5.1857e-15, 3.2236e-14, -1.2228e-14, 7.5274e-14,\n -3.3422e-13, 5.4170e-14, 5.1692e-14, -8.6064e-15, -3.0469e-15,\n 4.1240e-13, 6.7986e-14, 3.8025e-14, 1.8210e-14, -1.2680e-14,\n 6.3285e-13, 6.4166e-16, 6.5584e-13, 9.5555e-15, 6.8940e-15,\n 5.9338e-14, 1.8470e-13, 3.3041e-13, -3.0462e-17, 2.6888e-15,\n 2.0956e-14, 6.1415e-14, -1.5150e-15, 3.8661e-13, 9.8279e-15,\n 2.9333e-15, -1.7118e-14, 1.9620e-14, 3.6669e-13, -1.0432e-15,\n -4.5210e-15, -2.0159e-14, 5.3726e-15, 5.1770e-15, 2.6009e-13,\n 2.5378e-13, 5.9819e-16, 3.2256e-14, 8.3814e-15, 7.2453e-15,\n -7.8246e-17, -9.1770e-15, -3.1366e-16, -8.8012e-15, 2.9352e-13,\n 2.0526e-15, 5.3890e-14, -1.8834e-14, 1.8035e-14, 7.8348e-14,\n 1.9582e-14, -9.6968e-15, 1.1654e-14, 1.2443e-15, -1.1714e-15,\n -8.1586e-16, 7.8539e-14, 1.1552e-13, 1.2995e-13, 7.9134e-16,\n 1.0862e-14, 2.2095e-15, 7.3364e-14, 3.8089e-14, 1.6518e-14,\n 1.7130e-14, 1.3374e-14, -9.1118e-14, -2.2011e-15, 2.4174e-13,\n 1.6743e-14, -6.6614e-17, 2.1124e-15, -8.9775e-15, -6.1297e-15,\n 1.8102e-14, 3.1768e-13, -5.5312e-15, 3.1141e-14, 5.8378e-14,\n 2.2386e-14, -3.4049e-15, -2.8123e-16, -2.4838e-15, 7.6374e-14,\n 1.9963e-14, 1.4179e-13, 9.1491e-15, 1.8000e-14, -1.4820e-14,\n -5.4944e-14, 6.5751e-14, 6.2901e-14, -3.2602e-15, -9.1344e-15,\n -1.3694e-13, -1.1601e-14, 4.1542e-15, 4.7879e-15, -7.1512e-15,\n 2.8185e-14, 4.1360e-14, -9.8629e-14, 3.3135e-13, 4.6451e-14,\n -3.1579e-14, 3.7159e-15, -5.7974e-15, -2.2186e-14, 2.3976e-15,\n 6.8883e-14, -6.6318e-16, 1.3290e-14, -2.7159e-14, -2.1785e-14,\n -4.6307e-15, -1.2757e-14, -8.3680e-16, -9.9973e-15, -8.6113e-15,\n -7.9857e-16, -1.2684e-14, 4.7250e-13, -3.4801e-14, 4.8261e-14,\n 4.6800e-14, 3.6269e-13, -4.9373e-15, 1.1188e-14, 5.3900e-13,\n -1.8081e-15, 3.7144e-14, 1.4083e-14, 4.7117e-13, 1.7946e-14,\n -7.3887e-15, 9.7690e-15, 1.1422e-13, -3.2260e-14, 4.7769e-15,\n -6.6757e-16, 8.9204e-14, 5.2979e-13, 9.3833e-15, 7.0028e-14,\n 1.8007e-13, 2.1424e-13, -3.4214e-15, -5.3608e-15, 1.8234e-13,\n -7.2997e-15, 3.5490e-15, -1.1621e-14, 1.5618e-14, 1.7307e-14,\n 2.9843e-13, 1.6739e-13, 1.1418e-14, 6.9210e-15, -1.4733e-14,\n 2.2753e-13, 1.4060e-13, -7.7452e-16, 7.5175e-16, 1.5939e-14,\n 6.3683e-14, 1.3786e-13, 1.9832e-14, 5.7640e-15, 4.7399e-14,\n 4.5948e-14, -9.7522e-16, 2.4589e-13, 6.9859e-14, 3.1514e-15,\n 7.2453e-14, 4.7770e-14, 4.9077e-14, 4.4396e-15, 1.2862e-13,\n 2.5273e-14, 9.7994e-14, 4.0487e-14, -9.4995e-16, 1.8138e-13,\n 1.9220e-13, 1.0205e-14, 9.7802e-14, -3.5135e-15, 1.2490e-14,\n 9.6144e-13, 5.0645e-14, 1.4057e-14, 2.8627e-15, 2.7350e-13,\n 1.6058e-13, 6.0097e-15, 1.5427e-13, 1.8783e-13, 2.2691e-13,\n 1.4152e-13, -7.8952e-16, 1.0934e-13, -2.6978e-15, 4.0391e-14,\n 2.3030e-14, 1.0059e-14, -2.7520e-15, 4.1503e-14, -8.6231e-14,\n 1.9007e-15, 2.7262e-13, 6.1865e-14, -1.6930e-14, 1.9912e-13,\n 1.1405e-14, 1.7623e-13, 7.3960e-14, -9.5231e-15, 6.2173e-14,\n 3.5790e-13, -1.1990e-14, -1.9740e-14, 3.6275e-14, 2.1294e-14,\n 3.9721e-15, -1.6761e-14, 1.3007e-14, 8.7482e-15, 9.8899e-14,\n 5.9191e-14, 4.4189e-14, -1.1724e-14, 9.9655e-14, -4.5237e-16,\n 4.7472e-13, 2.0854e-13, 4.5954e-14, 5.7047e-14, -1.5219e-14,\n -3.3998e-15, -1.1035e-13, 1.3039e-13, -3.0222e-15, 9.9703e-15,\n 8.4816e-16, 1.4866e-13, 5.1153e-14, 1.1204e-15, -1.8863e-15,\n 2.1085e-13, 7.5721e-15, 5.2727e-15, 1.9162e-14, -2.6671e-15,\n 1.6155e-14, 2.3354e-14, -1.9218e-15, -1.2966e-14, 2.5029e-13,\n 2.6868e-14, -2.0425e-14, 3.7368e-14, 3.7537e-15, 1.4304e-14,\n -4.9641e-16, 3.5678e-14, 1.3337e-14, -3.8375e-16, 2.1043e-13,\n -4.6991e-15, -1.6785e-14, 5.1353e-16, -5.5508e-15, -8.6320e-15,\n 1.9286e-13, -1.0820e-13, -1.3483e-14, -2.5745e-15, 2.1409e-13,\n 3.9784e-14, 9.3885e-14, -1.4496e-15, 3.8405e-13, 2.0763e-13,\n 2.0290e-14, 2.3415e-13, -1.3295e-15, 4.5118e-13, 2.0274e-13,\n 3.5304e-14, -3.5585e-15, 2.2743e-14, -1.0095e-14, 4.9943e-14,\n -7.0817e-16, -1.3133e-15, -2.0285e-14, 1.7994e-14, -2.4411e-15,\n 1.1105e-13, -2.6397e-16, 2.8164e-15, 1.9391e-14, -3.0259e-15,\n -1.2662e-14, 1.7089e-13, 5.2977e-14, 1.0860e-14, 1.3764e-14,\n 1.1635e-16, -1.2814e-14, -2.4200e-14, -4.5340e-15, 8.1242e-16,\n -6.6486e-15, -1.5769e-15, 1.6746e-15, 5.8495e-14, 1.4531e-14,\n 5.5699e-15, 6.0061e-15, 1.0407e-15, 1.5520e-13, 4.8008e-15,\n -4.6788e-15, 1.8588e-13, -5.0624e-14, -4.3191e-14, 9.5598e-14,\n 8.7760e-15, 3.7421e-14, 3.4637e-13, 7.6610e-14, -8.5289e-16,\n -1.8217e-15, 1.2938e-13, -4.9188e-15, 2.2905e-13, -3.7393e-15,\n 6.8656e-14, 8.7072e-15, 4.0898e-16, 2.3360e-14, 1.2788e-13,\n 9.1900e-15, 4.1403e-14, 1.8224e-14, -6.9260e-14, -2.1030e-14,\n 1.7823e-13, 1.9391e-14, -1.1989e-14, 6.0875e-15, -6.0995e-15,\n 6.9723e-14, -3.9435e-15, 8.5788e-14, -1.1088e-15, 7.5637e-14,\n -4.8958e-15, 2.3987e-14, 5.9228e-15, 5.8741e-14, 2.8236e-14,\n 1.1899e-13, 1.7065e-13, 3.1974e-14, 1.0904e-13, -1.4066e-14,\n 1.0505e-14, -2.2166e-15, 1.1127e-15, 1.1298e-13, 3.0176e-14,\n -2.3063e-15, 3.3101e-13, 1.2827e-13, 4.6477e-16, 1.4104e-13,\n -7.7985e-15, -4.0256e-15, 1.9142e-13, 3.9367e-15, 6.4541e-15,\n 7.4926e-16, 8.4382e-16, -2.1574e-15, 1.3896e-14, 5.8921e-14,\n 9.3666e-14, 2.9089e-14, 3.5552e-15, -7.8026e-15, -7.7224e-15,\n -3.8831e-15, 3.1740e-14, 5.4206e-15, 3.9468e-15, 2.1732e-13,\n -8.8150e-15, -4.5165e-14, 6.1219e-13, 7.2321e-15, 4.0624e-14,\n 1.9369e-13, 1.1774e-16, -5.5775e-16, 5.8949e-15, 8.6362e-16,\n 6.2828e-14, 2.9857e-14, 3.0837e-14, 1.3539e-15, -3.2306e-15,\n -1.2750e-15, 3.1357e-13, 3.5414e-13, 2.2651e-13, 4.5527e-14,\n 6.7844e-16, 7.7001e-14, -1.5135e-15, -1.8719e-14, 7.2779e-14,\n 4.2822e-14, 1.1741e-13, 3.6188e-15, 2.0818e-13, 8.9872e-14,\n 3.5126e-14, -4.8507e-15, -5.5154e-15, -1.0270e-13, 6.6399e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.2321e-17, 2.9371e-15, 1.1258e-16, 8.4651e-16, 1.4799e-17, 4.1639e-16,\n 3.5889e-15, 6.0489e-17, 1.3306e-16, 4.2287e-16, 3.5477e-15, 1.7093e-17,\n 2.5725e-17, 6.6714e-15, 1.7610e-17, 4.8329e-16, 4.3447e-17, 9.4123e-16,\n 3.0566e-17, 3.1291e-15, 6.5206e-18, 6.2928e-17, 7.8364e-17, 8.8591e-18,\n 1.0394e-17, 1.1432e-17, 1.6689e-16, 2.0659e-17, 5.3071e-17, 2.7680e-17,\n 7.4524e-16, 8.0333e-15, 6.0488e-15, 3.4020e-17, 3.2309e-17, 1.1107e-17,\n 3.7813e-15, 1.3519e-16, 1.7819e-16, 1.2105e-17, 8.9575e-18, 9.8717e-16,\n 4.1122e-16, 3.3154e-15, 1.2993e-15, 7.4370e-16, 2.2543e-14, 2.4933e-16,\n 6.0520e-18, 2.8135e-17, 2.8509e-15, 7.4984e-17, 1.5633e-15, 1.4088e-15,\n 1.1942e-15, 3.1688e-15, 5.1846e-18, 7.2372e-17, 3.1376e-16, 2.8884e-16,\n 6.7814e-15, 2.9364e-16, 1.9019e-16, 1.4719e-15, 1.9231e-17, 8.0422e-17,\n 2.1681e-15, 3.9926e-16, 9.0325e-17, 6.1761e-17, 1.0221e-15, 4.0436e-16,\n 1.7979e-17, 1.0438e-15, 6.7539e-16, 7.5578e-16, 3.7457e-16, 6.8828e-17,\n 3.4683e-17, 3.6572e-15, 2.5463e-17, 1.0535e-16, 1.9937e-15, 2.6892e-17,\n 6.5615e-16, 3.0099e-17, 2.6847e-15, 3.2245e-16, 5.9051e-16, 8.5166e-16,\n 4.4476e-17, 9.5959e-16, 1.2119e-16, 5.0414e-16, 2.9811e-17, 1.0088e-15,\n 5.0319e-17, 3.6456e-18, 6.2260e-17, 1.5725e-15, 4.9761e-16, 1.7656e-16,\n 2.0898e-17, 9.3270e-16, 1.3642e-15, 1.3111e-16, 3.2365e-17, 8.9858e-16,\n 1.9422e-16, 5.0035e-17, 4.4942e-17, 4.2401e-17, 5.4854e-16, 7.4384e-17,\n 2.0971e-17, 5.4349e-15, 2.5246e-15, 3.1160e-17, 3.4085e-17, 1.3800e-15,\n 1.0006e-14, 6.1877e-15, 1.2972e-16, 1.0756e-16, 6.1915e-16, 9.3549e-15,\n 8.6217e-18, 6.2338e-17, 3.7500e-15, 2.2068e-16, 3.3264e-16, 8.2370e-17,\n 1.4091e-16, 3.0707e-17, 2.4830e-17, 6.7190e-17, 5.3974e-15, 4.1104e-17,\n 4.7522e-16, 1.1942e-16, 1.2601e-15, 2.6012e-17, 3.1117e-15, 9.9194e-18,\n 4.0197e-17, 6.0188e-15, 1.8892e-15, 7.0760e-15, 5.1424e-17, 1.0976e-17,\n 7.4068e-17, 6.2087e-15, 4.7543e-17, 2.0787e-17, 7.1173e-16, 6.3509e-16,\n 1.1384e-16, 3.0353e-15, 6.0296e-15, 3.0355e-15, 1.3150e-17, 1.6038e-17,\n 1.1325e-16, 1.3038e-17, 4.6972e-16, 4.2281e-16, 1.5649e-17, 1.7340e-15,\n 6.3040e-15, 2.6587e-16, 8.7330e-17, 1.3475e-15, 3.7650e-15, 3.4050e-15,\n 3.5347e-14, 1.0620e-15, 2.2044e-15, 1.1912e-15, 1.1052e-17, 1.2502e-17,\n 7.8977e-18, 4.9484e-16, 2.5806e-17, 4.6598e-15, 4.6171e-18, 8.7136e-15,\n 7.2671e-15, 2.4745e-15, 1.4710e-16, 1.3304e-17, 2.1687e-17, 3.6492e-15,\n 1.4967e-16, 1.2222e-16, 1.2447e-16, 2.5232e-16, 4.4903e-16, 4.6561e-16,\n 5.3449e-17, 1.0216e-15, 4.3655e-17, 7.8386e-18, 5.4164e-16, 7.8361e-18,\n 2.2401e-16, 1.0840e-14, 4.4622e-17, 5.6374e-18, 3.1259e-16, 5.7969e-17,\n 7.9790e-16, 3.1724e-15, 6.5218e-17, 4.1996e-17, 1.5993e-16, 1.4725e-14,\n 2.1140e-17, 3.8879e-16, 1.8501e-17, 2.7084e-15, 8.1672e-16, 1.2789e-14,\n 6.6299e-15, 2.7733e-17, 7.8675e-17, 1.0572e-17, 1.4764e-15, 7.3306e-18,\n 2.4949e-15, 6.4992e-17, 3.7747e-18, 4.6371e-15, 8.6477e-16, 4.1660e-15,\n 9.1897e-18, 5.6682e-17, 8.4780e-15, 1.1185e-15, 3.0852e-16, 6.3684e-15,\n 5.0751e-16, 3.7062e-17, 1.2069e-15, 9.3879e-17, 1.0651e-16, 6.0104e-18,\n 1.7492e-16, 1.6259e-17, 3.0267e-16, 4.9881e-15, 2.3936e-17, 2.1654e-16,\n 1.4057e-15, 1.0888e-16, 2.6062e-15, 6.7292e-16, 7.2085e-16, 3.9820e-17,\n 9.8555e-16, 1.3345e-17, 3.2973e-17, 7.9630e-17, 1.2981e-15, 1.4088e-15,\n 3.4784e-17, 1.3356e-15, 3.1947e-17, 6.1477e-15, 3.7048e-16, 2.2512e-17,\n 7.4684e-15, 4.6917e-18, 6.9738e-15, 2.2248e-15, 8.7967e-15, 1.0906e-15,\n 1.4470e-15, 2.9814e-16, 1.1484e-17, 4.2905e-17, 3.7640e-15, 2.2969e-14,\n 4.0756e-15, 1.4592e-17, 1.4961e-16, 2.0248e-17, 2.1968e-15, 3.8572e-17,\n 1.3646e-16, 4.1506e-16, 3.6097e-17, 2.3763e-14, 9.5221e-18, 1.2618e-16,\n 4.7044e-17, 6.5279e-15, 1.3905e-16, 1.6187e-15, 6.4377e-17, 1.3544e-17,\n 4.9162e-15, 3.4250e-16, 6.0229e-17, 8.8531e-16, 9.6699e-16, 9.8250e-17,\n 8.4430e-15, 3.3978e-15, 4.7445e-15, 7.0377e-18, 7.1556e-16, 8.8580e-17,\n 7.6860e-16, 2.4303e-16, 1.7231e-17, 1.3952e-15, 2.6778e-17, 1.1217e-15,\n 8.5422e-15, 9.3224e-16, 1.6307e-17, 1.5674e-17, 7.1903e-17, 4.4301e-17,\n 6.1408e-16, 2.6681e-15, 4.1935e-17, 1.1223e-15, 5.1629e-15, 2.2444e-15,\n 7.0539e-15, 1.4866e-15, 5.1311e-17, 1.8636e-17, 1.6362e-14, 4.7744e-17,\n 1.4759e-15, 1.6699e-17, 1.0532e-15, 9.2473e-17, 4.1744e-17, 4.1676e-17,\n 1.2309e-14, 5.0622e-15, 1.9360e-16, 2.7374e-16, 5.2405e-16, 2.5408e-15,\n 3.1232e-17, 5.6396e-16, 2.1669e-16, 1.1923e-15, 1.5179e-14, 3.2494e-16,\n 1.1302e-14, 3.0360e-16, 2.9132e-17, 7.2654e-17, 1.0441e-16, 2.2138e-17,\n 7.8059e-15, 2.6771e-16, 2.1004e-15, 9.1654e-17, 9.0798e-18, 9.6192e-17,\n 8.8677e-16, 6.0777e-17, 1.3146e-16, 1.3117e-17, 3.4779e-17, 5.8475e-15,\n 1.9005e-17, 2.1285e-16, 2.7551e-15, 1.4374e-14, 1.4857e-16, 5.5191e-15,\n 7.7012e-17, 9.3815e-17, 1.6398e-16, 3.8228e-16, 3.7055e-15, 4.8255e-17,\n 1.1159e-15, 2.6807e-15, 5.9623e-15, 2.2413e-15, 2.6697e-16, 1.4273e-14,\n 8.4136e-16, 4.6223e-17, 6.9165e-15, 5.7110e-17, 3.8428e-17, 6.6492e-15,\n 6.3085e-17, 2.4550e-14, 5.2363e-17, 9.3401e-17, 5.2936e-15, 6.2144e-17,\n 1.2703e-17, 5.1413e-16, 6.0208e-17, 2.1767e-16, 4.3898e-17, 5.5826e-15,\n 3.6278e-16, 4.5413e-17, 3.3871e-17, 8.3240e-18, 7.3123e-18, 7.2742e-15,\n 4.3647e-15, 4.6343e-17, 3.9956e-15, 5.5084e-15, 9.1489e-18, 2.5447e-15,\n 3.5671e-16, 5.9745e-15, 1.1937e-16, 2.6347e-16, 1.8100e-16, 4.9339e-16,\n 1.7508e-15, 8.0946e-17, 2.7216e-17, 1.2065e-14, 1.1514e-16, 6.1178e-15,\n 3.6113e-17, 3.0916e-15, 2.8932e-15, 3.7562e-15, 3.6091e-15, 6.0994e-16,\n 5.9224e-16, 1.0839e-17, 7.6685e-15, 8.9387e-15, 5.8078e-15, 2.0254e-15,\n 1.3082e-16, 2.6415e-18, 1.7785e-14, 9.3303e-15, 1.5183e-16, 1.1701e-16,\n 1.6502e-17, 6.9043e-16, 8.4809e-17, 6.6097e-17, 1.3901e-17, 4.9042e-15,\n 2.0303e-17, 5.9294e-16, 3.1506e-17, 3.4052e-17, 1.9208e-17, 2.1719e-15,\n 6.1620e-17, 6.3154e-16, 6.1872e-15, 1.6404e-14, 1.8619e-15, 4.3894e-16,\n 3.5444e-17, 3.1604e-17, 1.3760e-17, 3.1546e-17, 6.9004e-15, 3.6937e-17,\n 8.8983e-15, 5.7397e-16, 1.0957e-15, 2.3056e-16, 1.6670e-16, 3.3764e-16,\n 1.1458e-15, 6.9568e-15, 5.9296e-17, 1.4280e-16, 5.2885e-17, 2.2239e-15,\n 2.9700e-15, 2.7199e-15, 1.8422e-16, 5.1459e-15, 3.5170e-16, 6.1416e-16,\n 6.1599e-18, 1.6856e-14, 5.6619e-15, 7.1252e-16, 9.4817e-17, 1.2744e-17,\n 1.9820e-16, 7.9035e-15, 8.0277e-17, 1.6842e-15, 9.1713e-16, 5.1016e-15,\n 1.2558e-16, 1.0565e-15, 2.3128e-17, 1.6848e-16, 3.9558e-16, 3.8660e-17,\n 4.7496e-16, 1.0224e-15, 1.9628e-15, 1.7510e-16, 1.4249e-16, 2.4748e-16,\n 2.2816e-16, 1.8178e-15, 3.2693e-17, 5.3348e-17, 5.3304e-15, 9.5496e-18,\n 2.9928e-14, 1.8403e-15, 5.3002e-17, 3.1169e-15, 6.4735e-17, 1.6124e-15,\n 7.2418e-15, 8.5200e-15, 3.0809e-16, 8.6109e-18, 2.7533e-15, 1.0056e-14,\n 2.6095e-15, 5.0722e-18, 5.5639e-15, 2.4128e-15, 6.7958e-15, 4.5088e-16,\n 3.4002e-17, 9.9374e-16, 8.4755e-17, 2.7462e-15, 2.1346e-15, 1.5885e-15,\n 9.6643e-18, 9.0769e-17, 1.4908e-17, 3.5812e-16, 6.5294e-16, 5.5351e-16,\n 1.4667e-17, 5.8315e-15, 6.1993e-15, 6.0446e-16, 9.0597e-17, 3.7913e-18,\n 5.8711e-17, 1.3743e-17, 9.6858e-15, 3.0491e-17, 1.4113e-15, 1.1667e-15,\n 1.3404e-15, 9.4366e-18, 1.9728e-17, 1.5273e-16, 2.9058e-15, 4.6547e-17,\n 9.3008e-16, 3.6992e-15, 1.7932e-16, 1.6754e-16, 2.4966e-15, 1.1225e-17,\n 2.5658e-17, 1.4486e-17, 3.0037e-15, 3.2142e-17, 1.1883e-16, 3.8552e-16,\n 2.7323e-17, 3.9821e-15, 4.6172e-16, 7.5100e-18, 1.3029e-17, 2.4373e-16,\n 2.2130e-17, 8.4405e-16, 4.5845e-17, 6.0897e-18, 2.1147e-16, 1.2468e-15,\n 3.1715e-16, 2.1012e-15, 3.5357e-15, 4.1221e-17, 2.7694e-16, 3.0344e-16,\n 5.2583e-17, 3.6510e-18, 7.5884e-17, 4.1688e-16, 2.3438e-14, 2.3098e-15,\n 9.0700e-17, 1.1553e-14, 4.4200e-16, 7.4491e-17, 7.8076e-15, 3.5815e-16,\n 5.2018e-16, 4.5588e-17, 1.3968e-16, 6.9277e-16, 4.9808e-15, 1.6778e-15,\n 3.3678e-17, 4.9354e-16, 8.1859e-16, 3.6040e-17, 1.9194e-15, 2.2956e-14,\n 3.5638e-17, 1.2484e-15, 9.5358e-15, 1.9074e-17, 1.7711e-16, 1.4257e-16,\n 5.4790e-17, 1.1010e-15, 5.5446e-16, 7.5170e-16, 4.7696e-15, 2.0905e-16,\n 4.4217e-18, 1.3406e-16, 6.5791e-17, 1.8469e-16], device='cuda:0')" + }, + "49": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-2.7169e-14, 1.2770e-13, 8.8467e-14, 1.6960e-13, -4.9412e-15,\n 7.6987e-14, 5.7041e-14, 6.5473e-14, 1.4256e-14, 4.0901e-14,\n 4.3086e-13, -5.5422e-14, -2.2260e-14, 1.2666e-13, 2.3555e-13,\n 1.3212e-13, -4.1233e-14, 5.3939e-14, 1.6317e-13, 1.2818e-13,\n 4.9623e-14, -3.4608e-15, 2.9797e-14, 6.1524e-15, 2.9085e-16,\n -2.5217e-14, -4.5953e-15, 2.7522e-15, 1.2006e-13, 9.6788e-14,\n 2.8417e-14, 2.4512e-13, 1.4301e-13, -1.8921e-14, 2.1714e-13,\n 5.6509e-15, 1.0523e-13, 2.9630e-15, 1.7757e-13, 7.0556e-14,\n 3.2704e-14, 7.0240e-14, 1.6929e-13, -1.4003e-14, 1.8859e-13,\n 1.4014e-13, 7.0158e-14, 4.9264e-14, -2.3364e-14, -5.2396e-16,\n 9.1134e-14, -4.7343e-15, -2.9284e-14, 7.9480e-15, -4.8166e-15,\n 1.4051e-13, 1.0155e-13, -2.1532e-14, 4.0074e-14, 2.2686e-13,\n -2.8042e-14, -1.5815e-14, 2.3400e-13, 8.4276e-14, 2.1828e-13,\n -1.6324e-15, -9.6402e-14, 1.7840e-13, 1.1148e-15, -2.9756e-15,\n 2.6998e-13, -1.4294e-14, 5.8810e-14, 2.4381e-14, 8.5510e-15,\n -3.2439e-15, -1.2518e-14, -8.3930e-15, 6.1238e-14, 7.0705e-15,\n -1.7960e-15, 4.2723e-14, 2.9115e-13, 2.4932e-13, 5.1942e-15,\n 4.5139e-14, 2.5598e-13, -2.6145e-15, 5.4197e-16, 4.6160e-14,\n -2.1289e-14, 2.2114e-13, -3.2882e-14, -1.9763e-14, 4.4386e-14,\n 7.0141e-14, -2.5945e-14, 1.3167e-14, -1.0343e-14, 1.8593e-14,\n -4.3684e-14, -5.1113e-15, 1.5384e-13, 8.9749e-14, 1.9042e-13,\n -1.3043e-14, -1.4419e-14, 6.6728e-14, 3.1654e-14, 9.6374e-15,\n -1.0732e-14, -4.7466e-16, 1.5568e-13, 5.0118e-15, 2.2875e-15,\n 3.8517e-14, 5.7299e-14, 1.4871e-13, 1.0501e-15, 1.1098e-13,\n 1.8255e-13, -3.6430e-14, -1.4284e-14, -1.5996e-14, 2.7792e-13,\n 1.2832e-13, 3.6744e-15, -4.6848e-15, 1.0198e-14, 1.6472e-13,\n 6.7704e-14, -6.3903e-16, -1.4808e-14, 9.1044e-15, -2.9835e-14,\n 5.5567e-14, 6.0106e-14, -4.2356e-14, 1.2345e-13, 2.7696e-14,\n 2.2232e-14, 4.3768e-14, 1.3777e-15, 6.4113e-14, 1.0652e-13,\n 3.0408e-13, 1.3046e-13, 8.5857e-14, -3.0353e-14, -6.8060e-15,\n 6.4529e-14, 3.0304e-13, -1.5387e-14, -1.0523e-14, 2.4417e-13,\n 8.3013e-15, 7.7192e-14, 1.1127e-13, 2.1432e-14, 1.7230e-13,\n 5.8085e-14, 2.2707e-15, -7.5244e-15, 6.9293e-14, 8.0133e-14,\n 1.0256e-13, -1.2839e-14, 1.9084e-14, 1.4103e-13, 1.0346e-13,\n -2.5068e-14, 3.8341e-13, -3.8829e-15, 1.3087e-14, 5.0101e-13,\n -6.6483e-15, 3.2923e-13, 7.4317e-14, -2.5477e-15, 7.2647e-14,\n -6.8756e-17, 2.4661e-13, 9.0006e-14, 3.5829e-13, -5.6008e-15,\n 2.5314e-13, 2.0078e-13, 6.1999e-14, 3.2169e-14, -6.0046e-14,\n -1.1285e-14, 1.8936e-13, 1.0699e-13, -3.0817e-14, 9.6475e-15,\n 2.9015e-14, 1.4150e-13, 1.6077e-14, -1.7018e-14, 1.2805e-13,\n 2.9702e-13, 5.2309e-14, 1.2734e-13, 1.2110e-14, 5.0858e-14,\n -2.1256e-13, -3.5184e-14, 1.1577e-13, 2.4404e-14, 1.9511e-15,\n 2.9944e-13, 1.9832e-13, -2.5796e-14, -9.8185e-15, 4.0821e-14,\n 4.7960e-13, 8.4104e-16, 4.0089e-13, -1.1863e-14, 1.3458e-13,\n 6.1302e-14, 2.9650e-13, 2.5574e-13, -6.1240e-17, 6.0102e-15,\n -1.8926e-14, 1.5710e-13, 1.5380e-15, 3.6992e-13, 8.3493e-14,\n 3.5669e-14, 8.8035e-14, 3.9779e-14, 4.2343e-13, -7.2901e-15,\n -2.6119e-15, -2.0289e-14, 3.2693e-15, 5.7944e-14, 2.3072e-13,\n 3.4138e-13, -2.8741e-15, 9.9249e-14, -8.8707e-14, -5.1607e-15,\n 1.4751e-14, 4.0827e-14, 2.9066e-16, 6.3797e-14, 2.6521e-13,\n -2.4410e-15, 2.4308e-14, 1.3976e-13, -1.6168e-14, 8.3318e-14,\n 1.2484e-13, 3.0581e-14, -1.1144e-14, -1.3268e-15, 5.6664e-16,\n 5.4120e-15, 1.2906e-13, 2.0907e-13, 1.9354e-13, -9.1411e-16,\n 4.5990e-14, -5.0148e-14, 2.0519e-13, 1.8363e-13, 7.9188e-14,\n 4.3642e-14, -1.1506e-14, -1.4678e-13, 3.8029e-15, 1.3206e-13,\n -1.2555e-14, -3.3710e-17, -1.5411e-15, -4.3923e-14, 4.6871e-14,\n 1.4403e-14, 4.1298e-13, 7.9954e-14, 9.4140e-14, 7.9206e-14,\n -1.7988e-14, 7.1147e-15, 5.9372e-16, 2.4859e-15, 1.9163e-13,\n -1.7455e-14, 7.8818e-14, -8.5482e-15, 4.5731e-14, 1.2273e-14,\n -4.7890e-14, 8.8071e-14, 5.5447e-14, 1.1857e-15, 1.3430e-14,\n -1.6539e-13, -1.2589e-14, 1.7325e-14, -7.1957e-15, 1.5892e-14,\n -2.1059e-14, 3.0592e-14, -1.4131e-13, 2.9713e-13, 1.4833e-13,\n -2.9564e-14, -5.0215e-15, -3.8307e-14, 4.3280e-14, -1.2981e-15,\n 2.0645e-13, 1.0371e-14, 1.7714e-13, -3.9109e-14, 4.5760e-14,\n 4.9752e-14, 8.4477e-14, 9.2794e-16, 8.2844e-14, 3.1834e-14,\n -8.3918e-15, 6.7287e-14, 4.1257e-13, -5.8485e-14, 2.5238e-13,\n 2.1296e-13, 3.2609e-13, 2.3542e-14, -4.8908e-15, 6.4152e-13,\n 1.7164e-15, 1.4666e-13, -1.3871e-14, 4.4472e-13, -1.0888e-14,\n 6.4614e-14, 1.3768e-13, 2.0137e-13, -2.6350e-14, 1.6924e-14,\n -2.0754e-14, 1.3282e-13, 3.1149e-13, -3.2239e-15, 1.8851e-13,\n 1.2949e-13, 3.1479e-13, 7.2270e-14, 1.9356e-14, 1.5227e-13,\n 4.8619e-14, -2.2166e-15, 7.7001e-15, -7.9488e-15, 3.3805e-14,\n 2.6518e-13, 1.9162e-13, 4.4806e-14, -1.3986e-14, 1.5026e-13,\n 2.7771e-13, 2.3742e-13, 2.0541e-15, -8.4658e-16, 3.3152e-14,\n 8.3363e-14, 3.1008e-13, -1.6027e-14, 4.7379e-14, 4.8016e-14,\n 2.1770e-13, 2.8003e-15, 4.8650e-13, 7.7418e-14, 1.3379e-14,\n 1.5559e-13, -4.1039e-14, 1.8156e-13, -9.1616e-15, 1.7020e-13,\n 1.0992e-13, 1.9387e-13, 7.2589e-14, 6.2280e-14, 3.8256e-13,\n 2.2620e-13, 3.1752e-14, 6.3169e-14, 7.3947e-15, -8.1918e-15,\n 5.4148e-13, -3.0924e-14, 4.3496e-14, -2.6429e-15, 3.4007e-13,\n 1.2607e-13, 6.0962e-14, 9.9229e-14, 2.0230e-13, 3.0861e-13,\n 1.3925e-13, 2.6019e-15, 1.0462e-13, -1.1460e-14, 6.8239e-14,\n -1.2914e-14, -7.2219e-15, 2.1851e-14, 4.8217e-14, -1.0735e-13,\n -4.1831e-15, 3.4934e-13, 2.0980e-13, 4.0503e-14, 2.0830e-13,\n -9.7106e-15, 2.3656e-13, 5.6788e-14, 2.0516e-14, 1.6363e-13,\n 2.0547e-13, 1.3325e-13, 5.1772e-14, 5.4711e-14, -4.4959e-14,\n 1.7415e-14, 6.1946e-14, 5.9053e-14, -1.3134e-14, 1.4103e-13,\n 1.3376e-13, 4.9206e-14, 1.1912e-13, 1.8820e-13, -3.9393e-15,\n 2.4556e-13, 1.5816e-13, 5.5491e-14, 5.9762e-14, 1.2858e-13,\n 3.3614e-15, -8.2256e-14, 1.9735e-13, 3.0107e-15, -4.6523e-15,\n -2.5207e-16, 2.3459e-13, 1.4600e-13, -1.5620e-15, 1.5408e-15,\n 1.8985e-13, 8.7902e-14, -6.3311e-15, -1.3565e-14, 3.3995e-15,\n -1.2516e-14, 9.6188e-14, 9.8720e-16, 2.4591e-14, 2.8488e-13,\n 1.3386e-14, -1.5440e-14, -2.4176e-14, -7.1649e-15, -2.2818e-14,\n 9.8578e-16, -1.8748e-14, 3.7154e-14, 2.8145e-16, 1.9299e-13,\n 2.5266e-14, -2.5337e-14, 2.4605e-15, 1.3942e-14, 3.4831e-14,\n 2.1660e-13, -1.3327e-13, 4.8436e-14, 8.1018e-16, 3.4515e-13,\n 2.8019e-14, 2.7532e-13, 5.6043e-14, 3.5875e-13, 2.0798e-13,\n -1.6420e-14, 3.3102e-13, -3.7158e-14, 3.2492e-13, 2.2925e-13,\n 3.6465e-14, 1.2001e-15, -1.7685e-14, 3.9914e-15, 3.5058e-14,\n 5.5773e-15, 1.2106e-14, 9.6096e-14, 3.9576e-14, 1.4926e-14,\n 1.5614e-13, 1.1016e-15, 5.9856e-15, 9.6978e-14, 4.5245e-15,\n -2.9616e-14, 1.9149e-13, 9.8330e-14, -3.5999e-14, 1.5309e-14,\n 4.6659e-14, 3.6529e-14, -2.5144e-14, 1.6748e-14, -2.8680e-16,\n 1.3212e-14, 7.4462e-14, 1.0123e-15, 2.0492e-13, -1.0967e-14,\n -1.9745e-15, -4.0018e-15, 5.0055e-14, 1.4960e-13, 4.5479e-14,\n 3.7607e-14, 2.5001e-13, -4.5251e-14, 8.9720e-14, 1.1247e-13,\n -8.0906e-15, 1.5859e-13, 2.4869e-13, 6.9426e-14, 9.6964e-14,\n 3.6818e-14, 2.0255e-13, -8.2786e-15, 3.3899e-13, 2.3652e-15,\n 1.2003e-13, -2.9687e-15, -1.9993e-16, 1.3971e-13, 9.7291e-14,\n 6.2711e-14, -3.5436e-14, -1.4228e-14, -6.0994e-14, -1.0301e-14,\n 1.8242e-13, -9.9239e-15, 2.4416e-14, 3.2688e-14, 1.0085e-14,\n 1.0981e-13, 1.4719e-15, 2.2475e-13, 8.3555e-15, 4.7673e-14,\n 4.1629e-15, -1.6893e-14, 7.2881e-14, 1.5165e-13, -1.7476e-14,\n 2.9586e-13, 1.9776e-13, 2.2268e-13, 2.0105e-13, -1.3580e-14,\n -1.1187e-14, -1.1854e-15, -1.1181e-15, 1.9305e-13, 9.9951e-14,\n 4.1404e-15, 1.7985e-13, 1.2592e-13, 7.0423e-14, 1.2783e-13,\n 1.2330e-14, 5.5255e-15, 1.9045e-13, -2.5574e-15, 6.1549e-14,\n -3.6477e-17, -3.7118e-16, 1.7697e-15, 3.4237e-14, 1.1833e-13,\n 1.0851e-13, 8.5752e-14, 8.0887e-14, 2.2474e-14, 2.1534e-14,\n 1.2501e-14, -2.5462e-14, 4.4932e-14, 1.1458e-13, 1.7471e-13,\n 1.9590e-14, -8.6628e-14, 2.6689e-13, 1.3608e-13, 1.0422e-13,\n 2.4563e-13, -4.8459e-16, 3.1464e-15, 5.7595e-14, -3.5047e-15,\n 6.7826e-14, 8.8125e-14, 3.4395e-14, -5.2503e-16, 7.3179e-15,\n -2.0884e-15, 2.5306e-13, 3.4761e-13, 3.6152e-13, 6.0034e-14,\n 3.5601e-15, 2.1883e-13, 5.0754e-15, 6.6199e-14, 2.9586e-13,\n -3.0222e-14, 2.1790e-13, -2.3390e-14, 3.0165e-13, 1.0558e-13,\n -1.6006e-14, 4.3390e-14, 4.7300e-15, -1.7617e-13, -6.2089e-15],\n device='cuda:0')", + "exp_avg_sq": "tensor([9.9152e-16, 4.4181e-15, 1.4174e-15, 2.6693e-15, 3.2446e-18, 8.7751e-16,\n 7.3813e-16, 3.9325e-16, 4.1181e-17, 1.6061e-15, 6.3080e-15, 3.6480e-16,\n 5.1474e-18, 2.1706e-15, 3.8778e-16, 1.0352e-15, 1.0707e-15, 2.1518e-15,\n 2.7156e-16, 2.3516e-15, 1.8662e-16, 2.2739e-17, 9.3645e-16, 1.9687e-16,\n 2.9280e-16, 6.8155e-16, 5.7137e-17, 3.8965e-18, 6.7299e-16, 4.6515e-16,\n 1.9580e-15, 7.0258e-15, 4.3617e-15, 2.0848e-17, 3.2149e-16, 3.9637e-16,\n 3.4685e-15, 8.0021e-17, 1.1499e-16, 7.8099e-17, 1.1649e-17, 2.0186e-15,\n 1.2943e-15, 2.6192e-15, 6.4418e-16, 1.8146e-15, 8.6956e-15, 1.8088e-15,\n 6.9662e-17, 6.2665e-18, 2.2519e-15, 3.9347e-17, 1.2382e-15, 2.4791e-15,\n 5.7736e-15, 3.6851e-15, 1.4263e-16, 3.4535e-17, 1.5692e-15, 7.2351e-16,\n 3.0034e-15, 7.7533e-16, 8.1408e-16, 4.6164e-15, 2.4145e-16, 3.5608e-17,\n 2.5905e-15, 2.2302e-15, 2.8455e-17, 2.8271e-17, 1.7894e-15, 7.1757e-16,\n 2.3782e-16, 2.0473e-15, 2.9292e-15, 1.9642e-15, 1.0559e-15, 5.7722e-16,\n 1.7272e-15, 3.6962e-15, 5.4755e-18, 7.7980e-16, 3.4140e-15, 3.1026e-16,\n 2.1533e-15, 4.0195e-16, 3.7062e-15, 2.9020e-15, 1.5844e-15, 2.3621e-15,\n 1.7894e-17, 1.3118e-15, 4.0315e-17, 1.4773e-15, 1.1265e-16, 9.5575e-16,\n 5.6462e-17, 2.6198e-16, 3.3356e-16, 1.0997e-15, 1.4459e-15, 1.2846e-15,\n 7.4979e-16, 5.0681e-16, 2.0144e-15, 3.0611e-16, 1.4744e-16, 1.3087e-15,\n 1.5157e-15, 3.1804e-17, 9.1797e-16, 1.3605e-17, 4.4327e-16, 3.1875e-17,\n 4.0193e-16, 4.4166e-15, 2.1140e-15, 2.9607e-16, 4.4886e-16, 2.4729e-15,\n 7.2240e-15, 2.8021e-15, 8.1296e-17, 1.1289e-15, 1.3192e-15, 4.7133e-15,\n 9.5938e-16, 2.1436e-17, 2.7286e-15, 1.3307e-15, 6.5455e-16, 1.0096e-15,\n 5.9319e-17, 7.6211e-17, 8.6864e-16, 1.3981e-15, 5.1387e-15, 5.5777e-16,\n 9.1301e-16, 5.5162e-16, 1.5105e-15, 3.7138e-16, 3.6771e-15, 2.6466e-16,\n 5.2790e-16, 4.8295e-15, 3.0204e-15, 4.4162e-15, 2.3305e-17, 1.1692e-17,\n 3.8739e-16, 5.2834e-15, 2.8128e-17, 1.3243e-16, 1.4422e-15, 1.4190e-15,\n 1.0587e-15, 3.5152e-15, 4.1672e-15, 1.3454e-15, 1.0736e-16, 1.6967e-17,\n 5.1736e-16, 3.4670e-16, 2.4050e-15, 9.4397e-16, 1.6025e-16, 3.4452e-15,\n 5.3231e-15, 2.2605e-15, 4.4977e-17, 3.1183e-15, 3.9021e-15, 4.4748e-15,\n 1.7404e-14, 2.8732e-15, 3.3317e-15, 1.5738e-15, 4.0599e-16, 5.4620e-16,\n 1.8713e-18, 1.4965e-15, 3.2092e-16, 4.8654e-15, 6.5425e-16, 6.7115e-15,\n 3.0927e-15, 1.9282e-15, 8.5934e-16, 1.5880e-16, 7.0434e-17, 8.1676e-15,\n 9.2577e-16, 4.9355e-17, 3.6799e-17, 1.0661e-15, 1.8360e-15, 2.0054e-15,\n 1.0121e-16, 1.8255e-15, 5.5514e-16, 6.7999e-16, 1.2312e-15, 9.3857e-17,\n 5.2550e-16, 6.2658e-15, 6.0347e-16, 3.6692e-16, 1.3033e-15, 2.7911e-17,\n 2.7383e-15, 3.3274e-15, 3.4879e-16, 8.8173e-18, 6.3174e-16, 4.8415e-15,\n 5.5154e-16, 1.1293e-15, 4.8937e-18, 4.3331e-15, 1.1108e-15, 3.7444e-15,\n 4.3614e-15, 5.5298e-18, 6.8864e-16, 8.3670e-17, 2.0947e-15, 1.2212e-16,\n 3.5657e-15, 1.2364e-15, 8.9389e-16, 4.8539e-15, 1.5535e-15, 3.9951e-15,\n 2.9646e-16, 2.0264e-17, 4.0961e-15, 1.3173e-15, 2.0969e-15, 2.2288e-15,\n 1.0883e-15, 2.0662e-16, 4.0969e-15, 6.9414e-16, 2.9560e-16, 1.3948e-17,\n 5.3579e-17, 1.4318e-16, 1.5588e-15, 6.4401e-15, 3.1179e-16, 1.1597e-15,\n 1.8151e-15, 2.8393e-17, 1.8186e-15, 1.3497e-15, 2.4246e-15, 6.1338e-16,\n 1.9150e-15, 3.5261e-18, 1.0007e-17, 8.8136e-16, 1.8652e-15, 3.0387e-15,\n 4.9169e-16, 2.5002e-15, 2.8951e-16, 4.7300e-15, 1.3872e-15, 1.8393e-16,\n 2.8049e-15, 1.9668e-16, 4.8124e-15, 3.4022e-15, 2.4374e-15, 3.1189e-15,\n 2.3655e-15, 1.2693e-15, 4.1533e-16, 4.9897e-17, 1.4642e-15, 6.4029e-15,\n 4.1609e-15, 5.2975e-16, 3.3391e-15, 7.6586e-16, 1.3630e-15, 1.1578e-15,\n 7.9656e-17, 1.7150e-15, 1.2674e-15, 7.6781e-15, 2.3659e-16, 8.5470e-16,\n 1.4202e-17, 4.1602e-15, 1.9554e-15, 1.7365e-15, 4.3253e-17, 1.3415e-16,\n 3.4757e-15, 1.3962e-15, 1.2287e-15, 2.4328e-15, 1.5103e-15, 4.7251e-16,\n 1.0112e-14, 6.0616e-15, 2.7557e-15, 9.5865e-17, 1.5332e-15, 5.6566e-16,\n 9.2391e-16, 2.9426e-15, 4.6081e-17, 4.8837e-15, 4.2912e-16, 1.0563e-15,\n 3.1673e-15, 2.5614e-15, 3.8201e-16, 1.3685e-16, 3.2687e-17, 3.9054e-16,\n 1.1249e-15, 2.6253e-15, 7.0852e-16, 3.3649e-15, 4.6474e-15, 3.1515e-15,\n 5.7087e-15, 2.6104e-15, 2.8382e-17, 4.2801e-18, 1.0987e-14, 4.9102e-16,\n 5.7105e-15, 7.2590e-16, 3.2332e-15, 4.9933e-17, 3.3643e-16, 4.3463e-16,\n 5.3204e-15, 4.6243e-15, 8.8081e-16, 1.2642e-15, 1.8455e-15, 4.8057e-15,\n 9.0921e-18, 2.7119e-15, 1.2444e-15, 2.9234e-15, 5.2869e-15, 1.4089e-15,\n 6.7510e-15, 1.6795e-15, 1.4082e-16, 2.0509e-17, 2.7866e-17, 2.0167e-16,\n 4.6521e-15, 7.8736e-16, 1.8312e-15, 5.3043e-16, 4.5841e-16, 5.7027e-16,\n 1.3580e-15, 3.1641e-17, 6.0698e-17, 9.3166e-17, 9.0747e-16, 3.3076e-15,\n 1.6281e-16, 1.3790e-15, 3.2677e-15, 9.4520e-15, 8.7794e-16, 6.0019e-15,\n 1.6764e-15, 1.4752e-15, 4.3420e-16, 1.3687e-15, 3.2891e-15, 2.2481e-17,\n 3.0641e-15, 6.8114e-15, 3.5102e-15, 2.7666e-15, 1.4715e-15, 5.6253e-15,\n 1.0695e-15, 3.7852e-16, 1.8195e-15, 5.1072e-16, 3.5275e-16, 2.6644e-15,\n 1.9811e-17, 8.6611e-15, 1.5715e-17, 8.5405e-16, 3.3183e-15, 1.0388e-15,\n 1.4095e-16, 3.1595e-15, 1.0555e-16, 9.1952e-16, 5.1433e-16, 2.2295e-15,\n 9.6475e-16, 6.2530e-17, 2.1851e-17, 1.7158e-16, 1.4707e-16, 7.1895e-15,\n 2.1159e-15, 6.2155e-16, 3.1492e-15, 4.3914e-15, 7.8198e-16, 1.9620e-15,\n 1.2096e-15, 4.2678e-15, 9.2389e-16, 6.4942e-16, 1.2786e-15, 1.5284e-15,\n 2.7354e-15, 1.4411e-15, 8.6638e-17, 1.0987e-14, 1.5355e-15, 5.1997e-15,\n 5.9731e-16, 5.5572e-15, 1.6653e-15, 3.4427e-15, 4.4924e-15, 1.9770e-15,\n 1.0052e-15, 1.4776e-16, 2.4010e-15, 3.2492e-15, 6.8350e-15, 2.0871e-15,\n 1.3576e-15, 1.0138e-16, 9.8110e-15, 5.7722e-15, 6.2161e-17, 6.5496e-17,\n 4.2787e-18, 7.7104e-16, 1.3233e-15, 3.6615e-17, 3.3305e-16, 2.6288e-15,\n 5.1181e-16, 2.0184e-15, 4.2128e-16, 1.2305e-16, 3.3909e-16, 4.6516e-15,\n 1.4724e-17, 2.0868e-15, 4.4906e-15, 6.5030e-15, 1.8774e-15, 1.3839e-15,\n 1.4766e-17, 5.8094e-16, 1.3640e-16, 1.1013e-17, 5.9630e-15, 5.0659e-16,\n 6.4407e-15, 1.0205e-15, 1.6125e-15, 1.2508e-15, 1.1168e-15, 6.6545e-16,\n 1.6631e-15, 7.4151e-15, 1.7795e-17, 5.0955e-17, 1.1680e-15, 2.9567e-15,\n 3.9442e-15, 3.2854e-15, 8.4574e-16, 4.9189e-15, 1.5006e-16, 4.1768e-15,\n 7.2731e-17, 4.9472e-15, 3.3650e-15, 8.9753e-16, 2.6458e-17, 1.7249e-17,\n 6.0753e-17, 3.5263e-15, 6.4723e-16, 1.5155e-15, 1.8372e-15, 4.8374e-15,\n 9.9001e-17, 1.5550e-15, 3.7620e-16, 1.1831e-15, 5.2750e-16, 3.1482e-16,\n 2.2847e-15, 1.6633e-15, 2.9433e-15, 7.2009e-16, 8.2523e-16, 8.1807e-17,\n 1.2446e-15, 3.6603e-15, 4.2306e-16, 1.3714e-17, 3.9935e-15, 1.7165e-16,\n 9.4889e-15, 1.4229e-15, 1.2365e-17, 3.8094e-15, 6.3596e-16, 6.1517e-15,\n 3.3858e-15, 8.6223e-15, 7.3621e-16, 1.2580e-16, 4.7609e-15, 9.7344e-15,\n 3.5401e-15, 1.4882e-17, 2.1707e-15, 4.0144e-15, 3.4922e-15, 1.1469e-15,\n 7.2832e-16, 9.2527e-16, 6.7072e-16, 1.9777e-15, 3.1403e-15, 1.7752e-15,\n 1.9565e-18, 7.0570e-16, 1.5893e-16, 2.0920e-15, 1.5595e-15, 1.0697e-15,\n 4.9868e-16, 4.2239e-15, 3.5024e-15, 9.4328e-16, 4.0654e-17, 9.0075e-17,\n 3.7705e-16, 6.3321e-17, 9.6992e-15, 6.9474e-18, 3.0869e-15, 2.3807e-15,\n 1.2242e-15, 3.4745e-16, 4.6599e-16, 4.6924e-16, 3.4035e-15, 9.3893e-18,\n 7.1571e-16, 2.4734e-15, 5.2529e-17, 9.8205e-16, 3.0457e-15, 2.9235e-16,\n 5.7381e-17, 5.2172e-16, 3.1067e-15, 2.8595e-17, 3.6559e-16, 1.4298e-15,\n 5.4267e-16, 2.8520e-15, 1.1510e-15, 5.6816e-16, 2.7199e-16, 1.8838e-15,\n 1.4047e-16, 1.5472e-15, 1.1633e-17, 5.4648e-16, 1.1412e-15, 1.2513e-15,\n 1.3996e-15, 3.5068e-15, 2.3326e-15, 3.7083e-16, 1.5934e-15, 1.1953e-15,\n 3.5580e-16, 3.5563e-17, 5.1218e-16, 1.8224e-15, 1.5780e-14, 2.0772e-15,\n 1.1786e-15, 2.8403e-15, 1.1860e-15, 5.1538e-16, 3.4479e-15, 1.9615e-15,\n 3.6232e-15, 6.7550e-16, 3.4955e-17, 1.6244e-15, 4.7610e-15, 1.6775e-15,\n 1.1570e-15, 1.0013e-15, 1.6679e-15, 6.1880e-16, 1.7655e-15, 1.1027e-14,\n 3.6893e-16, 1.4581e-15, 7.0578e-15, 1.4749e-17, 8.7738e-16, 1.7982e-15,\n 4.1677e-17, 1.6834e-15, 5.6244e-16, 1.4937e-15, 4.0930e-15, 6.0157e-17,\n 8.1768e-17, 6.6389e-17, 1.2497e-15, 4.7947e-17], device='cuda:0')" + }, + "50": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -9.9871e-16, -1.6642e-17, ..., 3.3127e-15,\n 7.2285e-15, 9.2159e-17],\n [ 5.6052e-45, 4.5708e-16, 6.0424e-17, ..., 2.7183e-15,\n 7.6315e-16, 1.0058e-17],\n [-5.6052e-45, -8.7797e-17, 2.7599e-17, ..., 3.6484e-15,\n 9.2219e-15, 9.8067e-17],\n ...,\n [ 5.6052e-45, -2.4349e-17, -7.1448e-17, ..., -1.5698e-15,\n -3.3451e-16, -6.5102e-18],\n [ 5.6052e-45, -2.1885e-16, 2.8616e-18, ..., 1.2271e-17,\n -3.3828e-16, 6.5546e-18],\n [-5.6052e-45, -2.9308e-16, -1.2681e-17, ..., 4.9847e-16,\n 1.0758e-15, -2.5913e-18]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.4625e-16, 1.9905e-15, 2.1534e-15, ..., 1.1236e-17, 1.3212e-16,\n 1.1987e-15],\n [1.2010e-16, 1.4809e-16, 1.8885e-16, ..., 2.8660e-19, 1.0596e-17,\n 1.2841e-16],\n [3.7741e-16, 7.6303e-16, 7.1333e-16, ..., 2.5784e-17, 6.3295e-17,\n 2.8275e-16],\n ...,\n [6.4828e-16, 1.0017e-15, 1.1791e-15, ..., 1.0480e-18, 6.0171e-17,\n 9.6244e-16],\n [2.0445e-16, 3.5141e-16, 4.5894e-16, ..., 4.7077e-19, 1.9590e-17,\n 3.6214e-16],\n [1.2020e-16, 5.3603e-16, 6.3319e-16, ..., 2.2030e-19, 8.9890e-18,\n 3.2154e-16]], device='cuda:0')" + }, + "51": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.3837e-11, 5.0219e-12, 5.6056e-12, -1.1649e-12, -4.2948e-12,\n -8.6365e-13, -1.0417e-12, 1.0913e-12, -2.2951e-14, -1.1205e-12,\n 9.5589e-13, 1.4352e-12, -1.5690e-12, 4.2550e-12, 1.9508e-12,\n 3.4611e-12, -4.9496e-13, -2.0287e-12, -9.8946e-13, -3.4745e-13,\n -8.7593e-13, 4.2884e-12, -3.3523e-12, 9.2723e-13, -1.6257e-12,\n 3.6001e-12, 1.8541e-12, 4.4036e-13, 3.0862e-12, -1.0837e-12,\n -1.2192e-12, -1.4004e-12, 7.6930e-13, -6.3328e-13, 2.3807e-12,\n -1.7042e-12, 5.2338e-13, 1.2473e-12, 9.6998e-12, -1.1841e-12,\n -2.0183e-12, 4.6886e-12, -1.5413e-12, -1.7280e-12, -6.3082e-14,\n -1.0747e-12, 1.8066e-12, -1.2476e-12, -1.8070e-12, -1.0703e-12,\n -2.6250e-13, 2.7356e-13, 7.7170e-12, -5.7760e-14, 6.4865e-12,\n 4.3145e-13, 2.0289e-12, 7.0904e-13, 1.7334e-12, 2.1012e-13,\n -6.9935e-12, 3.4554e-13, -5.4463e-13, 1.8543e-12, 2.7479e-12,\n 2.7341e-12, -2.6813e-12, -1.3638e-14, -1.0005e-12, -9.7818e-13,\n 6.9816e-13, -1.7998e-12, 3.0210e-12, -2.6580e-12, -2.4950e-13,\n -1.7837e-12, -1.2733e-12, 7.4498e-13, 3.4037e-12, -3.3247e-12,\n -2.0050e-13, 2.8426e-12, -2.4022e-12, 4.5530e-12, 2.5539e-12,\n -1.3079e-12, -1.2065e-12, -1.4892e-12, 1.2149e-12, -2.0749e-12,\n 4.2164e-12, 4.2353e-12, -3.8572e-12, -5.9567e-12, -7.7354e-13,\n -6.7680e-13, -2.0348e-12, -4.0971e-12, -2.3409e-12, 9.7255e-13,\n -4.5654e-12, -1.1748e-12, -1.7500e-12, 4.6460e-12, 4.0718e-12,\n -3.1638e-12, -2.2800e-12, 1.8164e-12, -4.7603e-13, -1.5126e-12,\n 1.8909e-12, 4.0048e-12, -1.1174e-12, -3.7732e-12, 3.7567e-12,\n -1.1268e-12, -1.9080e-13, 1.2154e-13, -2.7796e-12, -3.7950e-13,\n 6.2558e-12, -6.7491e-12, 2.1504e-12, -2.9483e-12, 1.8172e-12,\n -4.7531e-12, 1.9669e-12, -4.9017e-12, -1.0795e-12, 7.5893e-13,\n 3.4872e-13, -1.2790e-12, -9.5397e-13, 2.0029e-12, -5.6799e-14,\n -1.7791e-12, 6.9312e-13, -8.9509e-13, 2.7256e-12, -1.3366e-12,\n -4.8346e-13, -8.9126e-13, -6.6102e-12, -1.8243e-13, 2.7782e-12,\n 7.6631e-12, 2.7588e-12, -1.9179e-12, -1.5314e-12, -9.1686e-13,\n 1.0427e-13, -3.5661e-13, 6.5446e-12, -1.1031e-12, 6.1620e-12,\n -7.1854e-13, -1.2715e-12, 3.0961e-12, -9.1589e-13, 2.2897e-12,\n -6.7778e-13, 1.0627e-12, -5.1815e-14, 2.1312e-13, -2.6628e-13,\n 4.0113e-12, -1.2130e-12, -2.7453e-13, -1.3099e-12, 3.1234e-12,\n 4.9720e-12, -1.5483e-12, -1.5791e-12, 4.3281e-12, 1.8779e-12,\n 1.4893e-14, 1.1179e-12, -2.9116e-12, -1.5188e-12, -1.9935e-12,\n 1.5663e-12, -1.4435e-12, 4.2041e-12, 4.9781e-12, -2.8723e-12,\n 1.3624e-12, 1.8419e-12, -1.0584e-12, -1.8596e-12, -2.0619e-12,\n -5.2172e-13, 5.7590e-12, 7.4411e-13, -2.4359e-12, -2.3759e-12,\n -7.5674e-13, -1.4042e-12, -1.3028e-12, -2.6975e-12, 2.1762e-12,\n 4.2872e-12, -2.1268e-12, 1.1958e-12, -3.2635e-12, -1.2017e-12,\n -1.4360e-11, 7.6060e-12, -3.9211e-12, 7.9350e-12, -3.0767e-12,\n 7.1835e-12, 5.4985e-12, 7.1075e-13, -1.8350e-12, -9.9437e-13,\n -4.8698e-13, 4.5495e-13, 4.4588e-12, -7.9600e-13, -1.3496e-13,\n 3.3959e-12, 2.9024e-12, 2.1439e-12, -2.3232e-12, -3.1667e-12,\n 2.6291e-12, -1.0949e-12, -3.3430e-12, -4.4220e-13, -1.0411e-12,\n 2.1311e-12, 1.4848e-12, -3.5081e-12, -1.6929e-12, 1.8347e-12,\n -2.0634e-12, -3.6233e-12, 2.6853e-12, 1.5116e-12, -1.5214e-12,\n -4.3125e-12, -2.3667e-12, -3.0274e-12, -3.6810e-12, 4.2799e-12,\n 8.1117e-14, -5.6570e-13, 1.9231e-13, 7.9407e-12, 2.5788e-12,\n 7.5146e-13, -4.4769e-12, 2.9667e-12, 7.5327e-12, -9.8893e-13,\n 1.5259e-12, 1.3192e-12, 2.9689e-12, -1.1461e-12, -1.4827e-12,\n -3.3091e-12, -2.0409e-12, -1.2726e-12, -1.2511e-12, -1.5315e-12,\n -1.3841e-12, 2.6612e-13, -4.5749e-16, -1.2922e-12, -1.6168e-12,\n 4.4575e-12, -1.5280e-12, -7.5427e-13, -9.1584e-13, 4.8765e-13,\n -6.9127e-13, -1.8810e-12, -1.3576e-12, -1.3893e-11, 3.4232e-13,\n -2.1345e-12, 4.0785e-12, -4.0795e-13, -4.0560e-12, -1.5823e-12,\n 4.6451e-13, -4.0444e-12, -1.2869e-13, -1.2294e-11, 3.0924e-12,\n 8.7709e-13, -2.7929e-12, -1.4306e-12, -4.0065e-13, -1.0244e-11,\n -4.1473e-12, -1.0486e-12, -1.3072e-12, -3.0514e-12, 1.0346e-13,\n -8.1490e-12, -1.3087e-12, -2.9840e-12, 3.9207e-13, -1.1696e-12,\n -8.3233e-13, 2.9951e-12, -1.1076e-11, -6.7057e-13, 4.1593e-12,\n -6.5199e-12, -1.4117e-12, -5.5927e-13, 8.1990e-12, -2.7626e-13,\n 7.4322e-12, -1.6092e-12, -1.2197e-12, -6.7243e-12, 1.1431e-11,\n 8.5695e-13, 4.8463e-12, -2.4151e-12, -9.3294e-13, -1.4042e-12,\n 1.3635e-12, 4.0966e-13, -8.0577e-14, -4.2469e-12, 3.4901e-12,\n 9.0277e-12, 4.9728e-12, -1.3084e-12, -2.2441e-12, -1.1467e-12,\n -2.6570e-12, 1.3533e-12, -1.3465e-13, 1.1109e-11, 3.7390e-13,\n -1.6845e-12, 4.1619e-12, 1.0436e-12, -1.2284e-12, -3.0973e-12,\n -5.2854e-12, -7.8670e-13, -9.9971e-13, 1.6723e-12, -9.6818e-13,\n -1.5948e-12, 6.3503e-13, -1.1011e-12, 2.5143e-12, -5.3924e-13,\n -1.2559e-13, 6.2026e-13, 1.2420e-13, 3.4396e-12, 2.8082e-12,\n -2.8742e-12, -1.3409e-12, -1.2065e-12, -4.8816e-12, 8.4685e-13,\n 6.8152e-12, -8.6592e-13, -1.5576e-12, -3.0430e-13, 5.9787e-13,\n -3.5670e-12, 5.5758e-12, -1.3124e-12, -8.1672e-13, -1.3832e-12,\n -1.3674e-13, -9.5791e-13, 4.2683e-12, -8.4268e-13, -3.7670e-12,\n -2.8599e-12, -6.1724e-13, 2.7559e-12, -1.3041e-12, 2.1528e-12,\n -8.2945e-12, -1.6734e-12, -1.1655e-12, -1.1616e-12, 6.2132e-12,\n -1.0553e-12, 1.5480e-12, -5.6630e-12, 5.3455e-13, 2.3570e-12,\n -1.7378e-12, 2.9147e-12, -1.1931e-12, -4.5345e-13, 1.4594e-13,\n -9.6846e-13, 4.3038e-12, 1.8295e-13, -2.0629e-12, 1.2787e-12,\n -1.1385e-12, 4.3221e-13, -3.6087e-13, -3.8232e-13, 3.2711e-12,\n 1.4954e-12, -7.6253e-13, -8.0411e-13, 2.4426e-12, -1.5410e-12,\n -1.1904e-12, 2.2096e-12, 3.5416e-12, 4.3422e-12, -1.2776e-12,\n -1.1944e-12, 1.6347e-12, 2.1278e-13, -1.3057e-13, 2.6117e-12,\n 2.0047e-12, 2.4145e-12, 3.9124e-13, 7.0135e-13, -5.8965e-12,\n 2.4908e-13, -1.2868e-12, -1.0792e-12, -5.4082e-13, -1.3145e-12,\n 1.8153e-12, -1.6901e-12, 4.4859e-12, 1.0255e-12, -3.0901e-13,\n -1.3030e-12, -2.4879e-12, -4.6399e-13, -6.4386e-13, -1.2887e-12,\n -1.0248e-12, -4.7682e-12, -1.7215e-12, 1.9367e-13, -1.2644e-12,\n -1.6627e-12, 5.4177e-12, -1.1753e-12, 2.8642e-13, -1.6529e-12,\n -1.4270e-12, 2.8141e-13, 2.5325e-12, -1.0975e-12, -1.1015e-12,\n 8.1178e-13, -9.6687e-15, -1.4958e-13, -1.4342e-12, 1.6253e-12,\n -3.4047e-12, -2.0919e-12, 3.4529e-12, -4.3885e-12, -2.7408e-12,\n -1.3715e-12, 3.1911e-12, -8.5299e-13, -6.9714e-13, 1.0196e-12,\n 1.1999e-12, 8.9100e-13, 9.3567e-13, -2.5304e-12, -2.0329e-12,\n -4.0085e-13, -8.5179e-12, 3.3606e-12, -5.4390e-12, 7.5782e-12,\n -1.4409e-12, 1.8770e-12, 1.7645e-12, -1.4253e-12, 1.6297e-12,\n 1.0141e-11, -8.8645e-13, -2.6487e-12, 4.6359e-12, 1.0923e-12,\n 6.1739e-12, -1.8140e-12, 3.0326e-12, -1.0384e-11, -6.8365e-14,\n -2.3096e-12, 1.3471e-12, 4.8505e-12, -1.4116e-12, -1.5713e-12,\n -1.9905e-12, -6.1563e-13, 2.4172e-12, -1.1526e-12, 6.8558e-14,\n -1.9090e-12, 1.2637e-12, -5.0653e-13, -4.0696e-12, 1.8687e-12,\n -6.5170e-14, 2.0417e-12, -7.9339e-12, 1.1169e-12, 3.7300e-12,\n 3.3531e-12, 6.8813e-13, -1.1862e-12, -2.6676e-12, 8.4414e-13,\n -1.4824e-12, -1.3467e-12, 4.8806e-13, 1.2169e-12, -1.4110e-12,\n -9.6884e-13, -7.5757e-13, -4.1340e-12, 1.1429e-11, 2.7390e-12,\n 6.4498e-12, -1.3222e-12, -1.3011e-12, -1.7413e-12, 1.7554e-13,\n -9.0522e-13, -1.2867e-12, -1.2795e-12, -1.1551e-13, -2.1479e-12,\n -2.6253e-13, 3.3903e-12, -1.3865e-12, -1.4676e-13, -1.3057e-12,\n -1.1860e-12, -2.0191e-12, -1.1269e-12, -2.2563e-12, -1.4254e-12,\n -1.3171e-12, -1.5151e-12, 1.9337e-12, -8.5451e-13, 2.1852e-12,\n 7.5010e-12, 7.8417e-13, 2.2001e-13, 8.3647e-13, -1.9147e-12,\n -1.8625e-12, 1.0143e-12, -1.4240e-12, -4.7333e-13, -4.7798e-13,\n -1.5103e-12, 2.4366e-12, 3.6888e-12, 4.1703e-12, -1.6463e-12,\n -1.4979e-12, -1.8136e-12, 2.2113e-12, 1.9673e-12, 4.0294e-12,\n -1.5945e-12, -3.4861e-12, 8.9591e-13, 2.7805e-12, 7.8703e-12,\n -2.4899e-13, -1.5178e-12, 2.6436e-12, -3.6285e-12, -8.4214e-13,\n -1.3396e-12, 3.3626e-12, 9.9885e-13, -1.6658e-12, -3.0889e-12,\n 9.1960e-12, -3.6444e-12, -1.3943e-12, -8.9661e-14, -1.2539e-13,\n 9.4910e-13, 2.2471e-12, 2.8905e-12, 2.4215e-12, -3.0751e-12,\n 3.2686e-12, -1.0866e-11, 1.9754e-12, -7.0681e-13, -9.5552e-13,\n -1.1574e-12, -1.6915e-12, -1.4137e-12, 5.6625e-12, -6.4581e-12,\n -1.8735e-12, -1.4398e-12, -3.7342e-13, -2.6960e-13, 2.1553e-12,\n -4.3437e-12, 2.2895e-12, 8.2237e-13, 9.2151e-12, -1.2188e-12,\n -3.7720e-12, -1.2076e-12, -2.9137e-12, -1.2418e-12, 2.9445e-12,\n 4.5678e-15, 3.7930e-12, 5.3257e-13, -1.1643e-12, -1.9249e-12,\n 3.6457e-12, -9.6244e-14, -1.8626e-12, 1.9789e-12, -1.6654e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([5.1162e-12, 2.6423e-13, 1.5934e-12, 5.4521e-14, 2.3001e-12, 1.5069e-12,\n 1.7607e-13, 5.2310e-14, 4.1817e-12, 2.0582e-12, 2.4426e-12, 2.8955e-12,\n 2.6551e-12, 3.3028e-13, 1.4694e-12, 1.3577e-12, 5.0563e-12, 3.2597e-12,\n 1.0316e-12, 9.2231e-13, 8.3423e-13, 1.6738e-12, 1.9914e-12, 4.2981e-12,\n 2.2515e-12, 3.9519e-12, 1.8385e-12, 1.0137e-13, 9.5288e-12, 7.0638e-13,\n 2.9833e-14, 1.2909e-12, 1.7885e-13, 4.1628e-13, 2.8381e-13, 1.0571e-12,\n 2.5695e-12, 6.3677e-12, 9.6084e-12, 2.0822e-13, 1.3367e-12, 6.3820e-13,\n 5.4926e-13, 2.5812e-13, 7.8214e-14, 3.0299e-13, 1.4248e-11, 4.4939e-13,\n 3.8566e-13, 7.4701e-13, 1.9571e-13, 1.9823e-13, 4.0383e-13, 1.9184e-13,\n 3.9117e-12, 3.1274e-13, 2.3376e-13, 2.7470e-12, 8.6660e-12, 1.2824e-12,\n 1.0893e-12, 1.1251e-11, 6.1764e-12, 1.2127e-11, 4.7620e-13, 2.5387e-13,\n 4.4778e-12, 5.3472e-12, 3.8570e-12, 1.4715e-12, 1.5718e-12, 1.1594e-13,\n 3.2242e-12, 4.5327e-12, 1.9225e-12, 6.6903e-12, 7.9313e-13, 4.5618e-13,\n 2.6267e-13, 3.7468e-13, 1.3490e-12, 4.1841e-12, 1.8526e-13, 1.1371e-12,\n 6.0155e-12, 6.6878e-13, 2.3935e-13, 6.4164e-13, 2.1079e-13, 2.2248e-12,\n 4.7227e-12, 3.0540e-12, 4.7624e-13, 3.7773e-12, 4.2541e-12, 1.4320e-12,\n 4.7051e-13, 1.8916e-12, 2.2966e-13, 2.4112e-13, 3.1600e-12, 1.8241e-12,\n 8.1855e-13, 1.1649e-12, 2.7978e-12, 1.2290e-12, 4.2791e-12, 2.5754e-14,\n 1.7602e-13, 3.5127e-13, 4.5786e-13, 1.4236e-12, 1.6142e-12, 4.3067e-12,\n 2.5632e-13, 1.9111e-13, 4.7800e-14, 3.5853e-13, 2.4420e-12, 6.7718e-12,\n 6.2115e-12, 5.1313e-14, 4.3370e-12, 3.0069e-12, 2.2744e-13, 3.2174e-12,\n 9.4436e-13, 1.6137e-12, 4.3552e-13, 1.9302e-13, 6.2232e-13, 8.3654e-13,\n 1.9355e-13, 1.1034e-13, 5.4164e-12, 2.9655e-12, 6.2940e-12, 1.4354e-12,\n 2.5915e-12, 2.3134e-13, 2.2843e-12, 1.3587e-12, 8.0971e-12, 8.9797e-13,\n 2.3107e-13, 8.7246e-12, 3.2194e-12, 1.9611e-12, 1.4444e-13, 1.8822e-12,\n 1.9969e-12, 2.3755e-13, 3.0164e-12, 4.8909e-12, 2.7121e-12, 3.5359e-12,\n 2.1263e-12, 1.6746e-12, 5.9601e-14, 5.8967e-13, 1.3085e-12, 2.7106e-13,\n 1.9745e-14, 7.9976e-12, 1.5864e-12, 1.8473e-13, 9.9838e-13, 9.3911e-14,\n 1.7883e-12, 3.9394e-13, 7.4100e-12, 4.6913e-12, 4.0146e-13, 2.1971e-12,\n 2.7611e-12, 7.4375e-13, 2.5586e-12, 2.8584e-13, 1.1403e-12, 5.5936e-12,\n 1.1805e-12, 7.5986e-14, 7.0358e-12, 1.1096e-12, 1.0717e-11, 1.4590e-11,\n 3.2511e-13, 1.1491e-12, 3.9868e-13, 3.3548e-12, 1.9360e-12, 8.5930e-12,\n 2.9789e-13, 1.5021e-13, 3.2522e-12, 8.3411e-14, 5.0861e-13, 5.5440e-12,\n 1.2350e-11, 2.1824e-12, 1.1749e-12, 1.2714e-11, 3.1430e-12, 1.1750e-12,\n 9.8971e-14, 3.7090e-12, 2.9903e-12, 4.0021e-12, 4.9975e-12, 2.3966e-12,\n 1.4802e-12, 4.6147e-12, 1.6525e-13, 1.3670e-12, 5.7386e-14, 4.6389e-13,\n 3.8297e-13, 1.4237e-12, 2.9148e-12, 3.2475e-13, 2.3794e-13, 1.5844e-12,\n 2.5213e-12, 7.5937e-12, 2.5998e-12, 7.5888e-14, 3.2112e-13, 9.4194e-13,\n 1.0281e-13, 2.8121e-13, 6.0231e-12, 1.9518e-12, 2.6498e-12, 3.2322e-13,\n 1.1713e-13, 3.6614e-12, 3.2152e-12, 3.5984e-13, 1.4528e-12, 3.9810e-13,\n 3.3606e-12, 3.0393e-12, 1.0347e-11, 5.0241e-12, 1.0382e-12, 3.6108e-13,\n 2.1073e-12, 4.0443e-12, 6.2010e-12, 6.6776e-13, 1.1359e-12, 3.0210e-12,\n 2.2259e-12, 4.8265e-14, 1.7018e-13, 2.3465e-13, 7.7086e-12, 9.4487e-12,\n 4.5622e-14, 5.1370e-13, 1.3856e-12, 1.4467e-12, 5.7583e-14, 1.0462e-12,\n 1.2699e-12, 2.2489e-12, 2.1147e-13, 4.5785e-13, 3.5131e-12, 5.2634e-14,\n 5.4042e-13, 2.3960e-13, 6.6704e-12, 1.9069e-12, 3.5467e-12, 5.8653e-14,\n 4.5889e-12, 1.2351e-12, 1.8966e-12, 4.4419e-12, 3.3703e-14, 4.6935e-12,\n 1.2594e-13, 5.9002e-12, 1.4004e-13, 2.6563e-13, 1.9318e-12, 1.3257e-12,\n 1.0510e-11, 3.4430e-12, 6.7059e-12, 1.0830e-12, 9.3439e-13, 1.7402e-13,\n 3.3695e-12, 4.6338e-12, 1.3627e-12, 9.2302e-14, 2.1087e-12, 4.9845e-12,\n 1.2665e-12, 6.2094e-13, 2.1622e-12, 1.6206e-11, 2.8622e-12, 3.0118e-14,\n 2.2197e-11, 2.6210e-11, 4.3857e-12, 5.3805e-12, 9.7249e-12, 2.6243e-12,\n 8.7251e-14, 1.0586e-11, 5.9103e-12, 2.5932e-13, 4.9196e-14, 8.4001e-13,\n 1.0887e-12, 5.4343e-14, 2.6059e-12, 1.9119e-12, 1.7801e-12, 2.2446e-12,\n 3.3379e-13, 4.8801e-12, 7.9694e-14, 5.4875e-12, 1.1370e-11, 4.8858e-12,\n 6.0228e-12, 3.0183e-12, 1.4360e-12, 8.1959e-14, 1.0594e-11, 1.0888e-12,\n 4.4690e-12, 4.0546e-12, 1.5633e-12, 8.3662e-12, 2.6495e-13, 2.7795e-12,\n 5.8217e-12, 1.2529e-12, 1.0039e-12, 2.1895e-12, 2.1069e-13, 9.7460e-14,\n 1.7879e-13, 1.3312e-13, 3.4892e-13, 4.9568e-12, 1.9759e-12, 1.2072e-12,\n 8.4720e-12, 2.4935e-13, 3.3484e-12, 8.7505e-14, 7.1204e-13, 1.7303e-12,\n 4.4287e-12, 1.3799e-12, 9.5021e-13, 5.0175e-12, 1.4995e-12, 5.7907e-14,\n 3.2186e-12, 2.7244e-13, 1.4456e-13, 9.0726e-13, 8.0773e-12, 5.9489e-13,\n 2.7192e-13, 6.7713e-13, 7.3683e-14, 2.0722e-13, 1.8074e-12, 5.8567e-12,\n 3.3126e-12, 7.1006e-12, 3.0936e-12, 3.8156e-12, 4.6956e-12, 1.1556e-12,\n 2.9759e-12, 1.5042e-11, 2.2443e-12, 8.3203e-13, 1.7441e-12, 2.4591e-12,\n 7.2885e-13, 2.9467e-12, 5.7583e-13, 2.6401e-13, 5.7712e-12, 3.1789e-14,\n 2.6694e-13, 2.9315e-12, 3.9033e-12, 2.3054e-12, 3.5705e-13, 2.3734e-12,\n 3.0254e-13, 6.3171e-13, 7.5280e-12, 1.8907e-13, 6.3441e-13, 1.3521e-12,\n 1.9432e-12, 4.1731e-12, 6.2530e-13, 3.8423e-13, 7.1719e-13, 2.6272e-13,\n 1.5310e-12, 4.7223e-13, 1.2448e-12, 1.1747e-11, 8.9979e-12, 2.6152e-12,\n 1.3986e-13, 4.3395e-12, 2.5655e-12, 9.3500e-13, 1.5210e-12, 1.9468e-12,\n 6.9273e-13, 1.0722e-12, 9.6050e-14, 1.4692e-11, 2.7470e-12, 6.6896e-14,\n 1.1241e-12, 2.6410e-13, 5.1859e-14, 7.1872e-12, 1.9545e-12, 2.8993e-12,\n 5.3935e-12, 1.3557e-13, 3.3095e-13, 2.1235e-12, 4.5563e-12, 2.1031e-12,\n 1.9692e-13, 1.9156e-13, 8.2143e-12, 9.2953e-12, 1.0248e-11, 5.4089e-14,\n 4.2831e-12, 1.3432e-12, 5.6605e-12, 2.1501e-12, 2.6927e-12, 2.2754e-12,\n 2.7650e-12, 2.2364e-13, 5.3256e-13, 5.3641e-13, 2.6327e-12, 4.1954e-12,\n 7.3472e-12, 1.2149e-12, 5.9509e-12, 8.2601e-12, 3.3472e-12, 4.2305e-12,\n 4.2391e-13, 6.1360e-12, 6.9544e-13, 1.1529e-12, 1.1947e-12, 4.7287e-13,\n 2.0455e-12, 2.0425e-13, 3.7378e-13, 7.5252e-13, 2.4508e-12, 4.2699e-12,\n 4.2767e-12, 5.2424e-12, 2.8891e-12, 1.1229e-11, 2.1085e-12, 1.8272e-12,\n 6.5815e-12, 6.2448e-13, 7.2357e-13, 6.8319e-13, 1.2159e-11, 7.1984e-12,\n 1.1796e-12, 5.2008e-12, 1.4349e-12, 1.4063e-12, 7.8394e-13, 1.8751e-13,\n 5.8782e-12, 2.5801e-12, 2.0466e-12, 5.2831e-12, 1.8306e-12, 1.8241e-13,\n 2.6608e-12, 6.0583e-12, 6.5374e-12, 3.6001e-13, 1.5501e-13, 5.2942e-14,\n 3.4569e-12, 2.5986e-12, 5.3380e-12, 2.7908e-12, 4.5879e-12, 7.4902e-12,\n 2.3437e-12, 1.9544e-12, 1.4102e-12, 8.0486e-12, 1.3362e-12, 1.3029e-12,\n 1.8805e-13, 1.5814e-12, 5.4190e-12, 9.4938e-14, 1.8497e-12, 3.6747e-12,\n 2.4481e-13, 1.3665e-13, 6.2892e-13, 6.6820e-14, 5.9956e-13, 1.1004e-11,\n 8.0650e-12, 1.0791e-12, 3.0536e-12, 1.1218e-13, 3.9966e-12, 2.2750e-12,\n 7.9085e-13, 2.1389e-12, 2.7868e-12, 3.1630e-12, 8.5334e-14, 2.1965e-12,\n 2.4674e-12, 1.6872e-12, 2.2582e-12, 4.4515e-13, 7.3499e-14, 8.1620e-13,\n 3.0983e-13, 2.5460e-12, 1.0146e-12, 1.9132e-12, 2.3422e-12, 4.3423e-12,\n 1.7992e-12, 9.5871e-12, 2.0467e-11, 6.9280e-13, 1.3146e-12, 4.7751e-12,\n 4.0144e-12, 3.6891e-12, 9.5260e-13, 5.5269e-14, 3.7248e-12, 3.0873e-13,\n 3.4418e-13, 4.0355e-12, 6.6549e-13, 2.2529e-12, 3.9848e-12, 2.0422e-12,\n 2.0547e-13, 3.2624e-13, 1.0382e-12, 5.5898e-12, 1.8553e-13, 5.5208e-12,\n 4.7739e-12, 2.2178e-12, 1.3221e-12, 3.5897e-13, 4.7638e-13, 7.6098e-13,\n 5.3155e-12, 1.4632e-12, 8.9302e-12, 3.0299e-12, 4.2398e-12, 1.2725e-12,\n 4.9921e-12, 9.5561e-13, 4.9978e-13, 2.3800e-12, 4.3893e-14, 4.6233e-13,\n 7.7965e-13, 2.3480e-13, 5.4756e-13, 1.5477e-12, 3.1951e-11, 2.2982e-12,\n 5.6457e-12, 2.6306e-12, 3.0227e-12, 1.3471e-13, 2.5286e-12, 1.4062e-12,\n 1.7373e-12, 5.3925e-12, 3.0380e-12, 3.6347e-12, 1.7085e-13, 5.8441e-13,\n 3.2384e-12, 4.6647e-13, 4.3248e-12, 9.0571e-14, 2.8509e-12, 5.7204e-12,\n 1.4221e-13, 1.5856e-12, 7.0020e-13, 6.6585e-12, 1.0774e-11, 3.0156e-12,\n 2.3473e-12, 8.1632e-13, 5.0962e-14, 1.1895e-13, 5.6078e-12, 1.8021e-11,\n 2.9775e-12, 7.4326e-13, 3.8342e-13, 2.6682e-13], device='cuda:0')" + }, + "52": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 9.2189e-13, 2.1024e-13, 3.9622e-14, 1.7279e-14, 6.3044e-14,\n 4.2005e-14, -2.3817e-15, -1.4167e-14, 8.0170e-14, -7.8993e-15,\n -2.1342e-14, 1.0077e-13, 1.0434e-14, -1.6599e-16, 1.2560e-13,\n 1.5031e-13, -1.5808e-14, -2.6910e-15, 1.0784e-14, -1.0354e-14,\n -1.1620e-14, 2.5007e-13, -5.1047e-16, 9.1699e-14, 2.3971e-14,\n 2.0187e-13, 3.0145e-14, -1.3123e-14, 9.2437e-14, 2.4900e-15,\n -1.7869e-15, 5.3960e-15, 1.9672e-14, -1.3823e-14, 7.6409e-15,\n 1.4991e-15, 3.2528e-14, -2.1486e-15, 2.2530e-13, 2.1276e-14,\n 6.3571e-16, 2.1216e-13, -2.6486e-15, 1.6125e-14, 3.9761e-14,\n 1.9795e-14, 1.9112e-14, 5.7932e-16, -1.2846e-14, -8.6658e-15,\n -1.9383e-14, 2.2597e-14, 1.4169e-13, 8.1651e-14, 3.3168e-13,\n 5.2640e-14, -1.0956e-14, 5.2111e-14, 3.5697e-13, -1.0340e-14,\n -6.2693e-14, 1.2400e-13, -8.2972e-15, 8.7076e-14, 5.6746e-14,\n 1.3208e-15, 1.6563e-13, -1.1630e-14, 7.5776e-14, -5.3045e-15,\n -2.5965e-14, 2.0542e-14, 1.1090e-13, 1.1945e-14, 2.9227e-15,\n 4.2912e-14, 3.4081e-15, 2.2330e-14, 2.7240e-15, 5.8713e-15,\n 9.6049e-15, 7.2018e-14, 6.2458e-14, 1.5834e-13, -4.5781e-15,\n 7.5991e-15, -8.9928e-16, 4.4074e-14, 3.4250e-15, 1.4462e-15,\n 2.6904e-13, 2.1641e-13, 3.7606e-14, -4.1184e-14, -8.8662e-15,\n 2.4076e-13, 1.6246e-15, -3.5256e-15, 3.5010e-16, -8.3831e-15,\n 8.8675e-15, -7.6277e-15, 6.0552e-14, 1.4750e-13, 1.0850e-13,\n -5.2055e-16, 2.1523e-14, 4.3047e-14, -1.4461e-14, -3.2464e-15,\n 6.7735e-15, 6.1504e-14, -7.5590e-15, 1.7161e-14, 4.3171e-14,\n -1.4680e-15, 7.9213e-15, -1.1730e-14, 5.0767e-14, 2.1478e-15,\n 2.6682e-13, -4.5369e-14, -1.5809e-15, -1.2532e-15, -1.8399e-14,\n 4.5846e-15, 1.4944e-15, -1.6039e-14, 4.8926e-15, 8.2390e-15,\n 8.4448e-14, -3.5974e-16, 4.6210e-14, 2.0786e-15, -1.1387e-14,\n 4.9484e-14, -2.2431e-15, -7.6652e-16, -1.4221e-14, 3.7075e-15,\n -1.7706e-14, -9.6835e-15, -5.5153e-14, -1.0197e-14, 5.0788e-15,\n 5.2393e-13, 2.8107e-13, 8.1933e-16, 7.6747e-14, 2.9615e-14,\n 1.9114e-15, 3.2504e-14, 1.1684e-13, -9.6522e-15, 3.2723e-13,\n 2.1244e-14, 1.1626e-15, 5.2026e-14, -8.4436e-15, 3.9712e-15,\n -1.1956e-14, 2.1856e-13, -1.3562e-14, -1.1904e-14, -6.1368e-15,\n -1.3859e-14, -3.4777e-15, -1.2192e-14, 2.4999e-14, 3.2282e-15,\n 2.7940e-14, 4.1215e-14, -5.4268e-16, 1.9641e-13, -2.2092e-14,\n 1.9694e-14, -2.8392e-14, -1.0263e-16, -6.3794e-16, 2.1588e-13,\n 2.9930e-14, 3.4064e-14, 1.5646e-13, 1.5360e-14, 6.8573e-14,\n 3.9535e-14, 8.8313e-15, 1.0376e-13, 6.9718e-15, 5.9270e-14,\n 1.0836e-15, 2.5341e-13, 2.4360e-15, -1.6414e-16, 3.0061e-14,\n -3.5820e-15, -4.5799e-15, 7.4852e-15, 1.0133e-14, 4.2685e-14,\n 7.8703e-13, -1.0861e-16, 2.6630e-13, 6.1914e-14, 3.9836e-15,\n -2.6349e-13, 1.5384e-13, 2.1565e-14, 5.0391e-14, 8.3856e-14,\n 2.1614e-13, 9.7906e-14, 6.9270e-15, 4.7194e-15, -6.1650e-15,\n -1.4500e-14, 2.4775e-15, -2.3893e-14, 2.2733e-13, 3.7749e-14,\n 1.8649e-15, 6.0460e-14, 1.3594e-13, 7.1149e-14, 5.1373e-16,\n 5.3118e-14, 1.3399e-14, 2.9527e-16, 6.1270e-14, 1.7872e-14,\n 1.9857e-13, -1.0655e-14, -2.0515e-15, 1.4556e-14, 2.2769e-15,\n 7.6574e-14, 4.2129e-14, -2.3716e-15, 2.4897e-15, -4.7363e-15,\n -4.8967e-15, 1.7351e-14, 2.5712e-14, 2.7652e-14, 1.1884e-13,\n 7.6326e-14, -7.6353e-15, 2.6183e-13, 2.8033e-13, 3.9519e-14,\n 1.1789e-13, -8.0374e-15, -1.2069e-14, 9.2947e-14, 1.2181e-14,\n 1.2627e-14, -9.3719e-15, 1.4890e-13, 3.9725e-15, 2.3137e-14,\n 3.2932e-17, -8.7749e-16, -3.9153e-15, 2.6950e-15, -7.0519e-15,\n 4.3295e-16, 4.4765e-14, -1.7675e-14, 7.7420e-15, -4.3537e-15,\n 4.6479e-13, 1.3681e-14, 1.5087e-13, 1.5981e-13, 1.0368e-13,\n 2.9507e-14, 1.8921e-13, -2.5232e-15, -3.1880e-13, 1.1884e-14,\n 1.1394e-15, -6.7838e-15, 7.1611e-15, 1.1134e-15, 8.9011e-15,\n 3.9644e-14, 1.8313e-14, 1.4572e-15, -2.8582e-13, 4.9369e-14,\n 9.3616e-15, 9.9066e-16, 8.9835e-15, -3.1810e-15, -1.8591e-13,\n -5.6736e-15, -4.5989e-15, -1.3558e-15, 1.0468e-15, 6.8224e-14,\n -1.0413e-13, 1.2435e-14, 6.2495e-16, -8.0913e-15, -1.5889e-15,\n -7.4442e-17, 1.7242e-13, -2.6886e-13, 1.1815e-15, 1.7522e-13,\n -6.3547e-14, 1.6184e-14, 5.1899e-15, 1.3972e-13, 2.9528e-14,\n 5.3566e-13, -4.4782e-15, -8.0884e-16, -5.6244e-14, 2.1734e-13,\n 1.4413e-14, 2.2585e-13, 1.3202e-14, -1.1348e-14, 9.9497e-17,\n 3.8837e-15, -1.0594e-14, 1.0323e-14, 2.9682e-14, 8.9911e-14,\n 3.9904e-13, 3.0110e-13, 1.3955e-15, 6.4264e-14, 2.7365e-14,\n 8.3631e-15, -2.0559e-14, -7.6667e-15, 2.3642e-13, 1.5052e-13,\n 5.1179e-14, 3.7691e-14, 3.3095e-14, -2.2013e-16, 2.4283e-14,\n -2.3973e-14, 2.9639e-14, 5.0611e-14, 6.5827e-15, -2.4640e-15,\n -2.3535e-15, 2.9686e-14, 1.3681e-15, 3.4221e-14, 8.6917e-16,\n 2.3686e-14, 9.9659e-15, -6.3809e-15, 2.5492e-13, 8.0599e-14,\n 1.4491e-14, 9.4276e-15, 1.1584e-14, -1.7604e-14, 6.4103e-14,\n 2.0696e-13, -6.5713e-15, -3.1343e-15, 3.7999e-15, 5.6364e-14,\n 2.3424e-16, 2.8016e-13, 5.3736e-14, 1.4607e-15, 1.1960e-15,\n -2.1114e-14, -1.0044e-14, -3.9048e-15, -1.8143e-15, 3.8440e-14,\n 1.2428e-14, 5.6309e-14, -1.4201e-14, 1.1945e-14, 2.5576e-13,\n -1.1546e-13, -2.8284e-15, -3.1529e-15, -3.4267e-15, 3.5873e-14,\n -4.6995e-15, 1.4289e-13, -1.2877e-14, 3.5389e-14, 6.4995e-14,\n 6.8401e-14, 5.9600e-15, 4.1928e-16, 2.1943e-13, -1.2800e-14,\n 1.5408e-14, 2.4216e-13, 1.9305e-14, 3.8491e-15, -3.6515e-14,\n -7.4477e-15, 3.5342e-14, 6.6074e-14, 6.1238e-14, 1.2855e-13,\n -4.2076e-15, -2.4763e-15, 1.1233e-15, -2.5317e-15, 1.6496e-15,\n -6.8848e-15, -5.3508e-15, 2.6752e-13, 5.7589e-14, -7.9485e-15,\n 5.2377e-15, 6.5514e-14, 2.5494e-15, -1.6575e-14, 3.2299e-14,\n 1.1733e-13, 5.3434e-13, -1.3864e-14, -1.8693e-14, -1.4616e-14,\n 1.4444e-13, 3.1358e-16, -3.0569e-15, -1.5420e-14, 3.7594e-14,\n 1.3613e-13, -3.7350e-15, 3.9566e-14, 3.9518e-13, 6.0642e-15,\n -6.8134e-16, 1.0094e-13, 5.0445e-15, 4.2372e-14, 2.8081e-14,\n 7.9328e-16, 4.0050e-15, 4.3146e-15, -9.3953e-15, 2.3414e-15,\n 7.1398e-14, 3.8834e-13, 4.2180e-14, 6.6858e-15, 1.0685e-14,\n -1.0022e-14, 1.1148e-13, 2.9832e-15, 6.8447e-15, 1.2855e-13,\n 6.8036e-15, -2.1403e-14, 1.9141e-13, 4.3035e-16, 6.6532e-13,\n 7.7750e-16, -1.1536e-15, 1.1112e-13, -6.0188e-15, 4.1159e-14,\n -2.2244e-15, 4.9553e-14, -9.1971e-15, -7.6332e-15, 2.5918e-14,\n 1.4679e-14, -1.8357e-15, 4.1098e-14, 1.1501e-14, 7.4195e-14,\n 4.5892e-13, -1.2184e-13, -4.7300e-15, -2.0706e-14, 9.0244e-14,\n -1.2758e-15, -1.3918e-14, -2.2421e-14, -3.1044e-16, 2.8593e-15,\n 1.2618e-13, -1.3520e-14, -4.2119e-16, 6.1602e-13, 5.7519e-15,\n 2.2545e-13, -1.1847e-15, 3.1681e-15, -1.8687e-13, 1.5109e-13,\n 2.5629e-15, -1.2465e-14, 7.3009e-14, -2.1362e-15, -2.4689e-15,\n 6.0768e-14, 2.3079e-14, 4.3263e-15, 2.5883e-14, 9.8559e-14,\n 1.4439e-14, 9.0994e-14, 2.7930e-13, 3.7169e-15, 1.8606e-13,\n 8.9544e-15, 9.4203e-14, -1.0621e-13, 5.6163e-14, 3.1156e-13,\n 2.8639e-13, 1.4764e-14, 3.5062e-15, 9.0716e-14, 1.5401e-13,\n -1.0368e-15, -3.4850e-16, 1.2742e-13, -7.4203e-15, 2.9139e-15,\n 1.0333e-14, -1.2687e-14, -3.8731e-15, 1.7944e-13, 2.1218e-13,\n 2.2144e-13, -1.7242e-15, 1.0400e-14, 2.3395e-14, -1.1717e-14,\n -1.1420e-14, 1.7411e-14, -4.7003e-15, 7.4354e-13, 4.6350e-16,\n 6.5697e-14, 6.5543e-14, -7.4628e-15, 1.5055e-14, -6.2412e-17,\n -4.1002e-15, 5.4446e-14, 6.7903e-15, 2.1079e-14, -1.9447e-15,\n 7.2452e-14, -1.6633e-15, 1.1977e-13, 1.5293e-14, 1.5256e-13,\n 3.8874e-14, 1.5978e-13, -1.1492e-14, 8.9958e-14, 1.8709e-15,\n 5.0856e-15, 1.0541e-14, 1.3635e-15, -1.2891e-14, 3.7317e-14,\n -5.4377e-15, 4.6381e-13, -1.8416e-14, 7.0044e-14, 7.2621e-14,\n -9.6171e-16, 3.6712e-16, 4.4985e-13, 3.4345e-14, 1.0292e-13,\n 1.2871e-15, 1.3006e-14, -1.9541e-14, 4.5904e-14, 1.7241e-13,\n -1.2299e-14, -2.3343e-15, 2.9833e-16, -2.7428e-16, -1.1212e-14,\n 6.5834e-14, 2.3627e-14, 1.9448e-13, 6.9719e-15, 3.7742e-14,\n 3.5163e-13, 4.7542e-16, 4.5962e-15, -1.4835e-14, -1.1579e-14,\n 9.5475e-14, 9.4779e-14, 9.6555e-14, -2.4304e-14, 1.7544e-15,\n 4.7435e-13, -1.8677e-13, 6.1667e-13, 5.2397e-15, 7.4354e-15,\n -8.2679e-15, -3.4626e-15, -4.4710e-15, 5.1292e-13, -4.2227e-14,\n 3.3501e-15, -4.3250e-15, 1.7464e-14, 1.9903e-14, 6.6368e-15,\n -1.0941e-14, 3.3820e-14, 7.6718e-14, 2.0441e-13, -4.8941e-15,\n -1.5134e-15, 4.3210e-15, -2.3254e-16, -3.3574e-15, 1.5405e-13,\n -1.5189e-14, 6.5572e-14, -8.5012e-15, -1.7903e-15, 6.5613e-14,\n 8.9189e-14, 3.7278e-14, -3.2027e-15, -5.4703e-15, 7.4667e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([1.1531e-14, 8.3668e-17, 9.1825e-16, 3.1400e-17, 7.4683e-16, 2.5275e-16,\n 7.5568e-18, 3.8712e-18, 5.1473e-15, 8.2938e-17, 1.4390e-16, 3.2737e-15,\n 1.2641e-15, 2.3414e-17, 1.4937e-15, 4.2925e-16, 2.2002e-15, 2.3386e-15,\n 1.6532e-16, 3.6370e-16, 1.2154e-16, 3.9623e-16, 1.3422e-16, 3.4459e-16,\n 2.9670e-16, 4.6687e-16, 5.9000e-17, 6.0768e-18, 1.5503e-14, 8.8634e-17,\n 1.4663e-17, 9.9497e-17, 8.6423e-17, 6.3477e-17, 2.3978e-17, 3.5066e-17,\n 6.4088e-16, 2.7343e-15, 5.1092e-15, 1.8880e-16, 2.5472e-16, 9.4375e-17,\n 2.3757e-17, 4.2115e-18, 4.0080e-17, 7.0556e-17, 5.3855e-15, 6.4187e-18,\n 1.2823e-16, 1.4590e-16, 5.3313e-17, 5.7395e-17, 5.2810e-17, 2.7567e-18,\n 7.1291e-16, 8.4958e-18, 1.1501e-17, 6.6382e-16, 2.2225e-14, 1.9623e-16,\n 1.9293e-16, 1.2007e-14, 3.1004e-15, 5.1488e-15, 7.8843e-17, 2.5764e-17,\n 3.6522e-15, 1.8901e-15, 9.9886e-16, 2.3540e-16, 1.1633e-15, 1.3595e-17,\n 5.4937e-16, 3.7079e-15, 3.6400e-16, 1.8822e-15, 1.9081e-16, 2.5383e-17,\n 9.6446e-17, 1.3254e-17, 3.9070e-16, 1.1739e-15, 3.7889e-17, 5.6303e-16,\n 2.4899e-15, 3.2489e-17, 7.8180e-18, 1.7697e-17, 4.3795e-17, 1.8161e-15,\n 1.3372e-15, 3.0762e-15, 3.1254e-17, 1.5138e-15, 3.6330e-15, 1.2144e-15,\n 1.8866e-17, 5.9545e-16, 4.1293e-17, 1.3060e-17, 9.5973e-16, 1.5566e-16,\n 5.8573e-17, 1.1190e-15, 7.9485e-15, 4.3618e-16, 1.3190e-15, 8.6502e-18,\n 6.7069e-17, 1.8591e-17, 7.1059e-17, 2.4610e-16, 9.4838e-16, 6.9720e-15,\n 3.9311e-17, 1.0075e-16, 5.1690e-17, 1.0296e-17, 1.8472e-15, 6.0427e-15,\n 8.9020e-15, 5.5640e-18, 2.4243e-15, 3.8711e-16, 3.8824e-17, 3.4791e-15,\n 2.7154e-16, 6.5034e-16, 3.9896e-18, 8.6375e-17, 3.4929e-17, 5.4647e-17,\n 1.6250e-16, 8.1466e-17, 8.9358e-16, 9.3047e-16, 4.4883e-15, 1.6845e-16,\n 1.7533e-16, 6.9806e-18, 2.7344e-16, 3.7178e-17, 7.4475e-15, 5.6374e-17,\n 2.2813e-16, 7.3605e-15, 1.5353e-15, 4.9060e-16, 4.0372e-17, 5.3831e-15,\n 5.5109e-16, 1.2614e-17, 2.4874e-15, 7.5594e-16, 8.3030e-16, 1.1759e-15,\n 2.7958e-16, 2.9855e-16, 8.7154e-17, 7.9656e-17, 1.1252e-16, 2.5495e-17,\n 3.0307e-17, 5.4861e-15, 6.5669e-18, 1.7994e-16, 3.6833e-16, 1.0098e-16,\n 5.8958e-17, 4.2690e-17, 6.6927e-15, 3.8282e-15, 1.1826e-16, 2.6840e-16,\n 2.9347e-16, 8.0261e-18, 5.8341e-16, 3.6702e-18, 7.4708e-17, 5.2721e-15,\n 4.2204e-16, 1.5906e-17, 1.9702e-15, 9.4585e-17, 1.0396e-14, 8.1082e-15,\n 3.5800e-17, 9.0556e-17, 1.0250e-17, 5.7283e-15, 3.4643e-16, 1.8698e-15,\n 4.2915e-17, 5.8332e-17, 9.6253e-16, 2.9814e-17, 1.1407e-17, 2.4789e-15,\n 3.4052e-14, 2.6225e-16, 1.0060e-15, 1.2328e-14, 7.6104e-15, 6.7112e-16,\n 1.9523e-17, 1.4090e-15, 8.7723e-15, 4.3193e-15, 3.5770e-15, 3.0831e-15,\n 4.6160e-16, 4.1024e-15, 3.7447e-17, 4.3578e-17, 4.1238e-18, 5.6945e-17,\n 2.0812e-17, 6.0248e-16, 3.0267e-15, 1.6026e-17, 2.6310e-17, 1.0945e-15,\n 3.6838e-16, 3.5471e-15, 1.1412e-16, 7.0111e-17, 3.2883e-18, 4.0510e-17,\n 1.6371e-17, 1.0450e-17, 2.6549e-15, 1.2660e-16, 4.8916e-16, 5.3661e-18,\n 4.0282e-17, 3.1799e-15, 1.6647e-15, 2.8163e-17, 1.6096e-17, 6.2215e-18,\n 1.3349e-15, 1.6577e-15, 3.8016e-15, 3.9143e-15, 4.3466e-16, 1.8159e-17,\n 6.0104e-17, 8.6208e-16, 9.4632e-15, 3.8123e-17, 4.1698e-16, 2.1416e-15,\n 3.9297e-16, 9.9070e-18, 1.1612e-16, 1.2354e-17, 9.5778e-15, 2.5079e-14,\n 6.1108e-17, 1.3855e-17, 3.7052e-16, 5.6130e-16, 5.7227e-17, 6.5101e-17,\n 1.0464e-16, 1.1460e-15, 4.3694e-17, 1.0888e-17, 2.0740e-15, 2.6655e-17,\n 2.3554e-16, 5.4202e-17, 1.0614e-14, 1.0979e-16, 9.0205e-16, 3.4376e-17,\n 2.8128e-15, 3.1372e-16, 4.1382e-16, 3.6379e-16, 2.0172e-17, 8.9758e-16,\n 4.2894e-17, 5.1422e-15, 2.5652e-16, 9.7424e-17, 4.1305e-16, 7.7153e-17,\n 4.9621e-15, 3.4085e-16, 2.5490e-15, 3.5721e-16, 3.1763e-16, 2.7246e-17,\n 2.0443e-15, 1.6653e-14, 4.9373e-17, 1.1396e-16, 4.9917e-16, 6.3458e-15,\n 1.9668e-16, 4.7972e-18, 2.6408e-16, 2.2741e-14, 1.7338e-15, 4.1907e-18,\n 7.1303e-15, 1.8375e-14, 5.2976e-15, 2.0018e-15, 5.3883e-15, 2.4133e-15,\n 4.6717e-17, 1.7355e-15, 3.3650e-15, 2.8958e-17, 7.4864e-17, 1.1897e-16,\n 3.3005e-16, 1.1949e-16, 2.2822e-16, 6.5534e-16, 6.8687e-16, 4.9591e-16,\n 2.4898e-17, 5.7559e-15, 4.6963e-17, 1.4567e-15, 1.2050e-14, 5.3072e-15,\n 1.5852e-14, 1.4892e-15, 3.3059e-17, 7.4375e-18, 7.7403e-15, 1.6620e-16,\n 5.1304e-16, 5.2052e-16, 2.0004e-16, 6.2302e-15, 2.1967e-17, 3.7365e-15,\n 9.8163e-16, 2.7056e-17, 7.2165e-17, 1.1000e-14, 2.0563e-17, 8.5582e-17,\n 3.8321e-17, 2.9512e-17, 6.5722e-18, 1.4052e-15, 6.2084e-16, 3.7622e-16,\n 1.4719e-15, 1.6326e-16, 3.5913e-15, 8.4770e-17, 1.5076e-17, 1.6054e-15,\n 4.4827e-15, 8.6759e-16, 4.7991e-17, 4.1993e-15, 3.5550e-15, 5.6334e-17,\n 2.1993e-15, 7.5301e-17, 1.5168e-16, 9.2275e-17, 5.7744e-15, 2.6531e-17,\n 8.6809e-18, 7.0407e-17, 7.7307e-17, 2.2309e-16, 3.5241e-16, 1.0127e-15,\n 2.5482e-16, 5.1194e-15, 2.3365e-15, 1.7783e-15, 3.2403e-15, 2.3495e-16,\n 3.4886e-16, 5.8975e-15, 9.1462e-16, 1.1050e-16, 1.2106e-16, 4.4647e-16,\n 1.3579e-16, 1.3684e-16, 4.2614e-17, 4.6018e-17, 6.4379e-15, 2.3264e-17,\n 4.4677e-17, 1.4223e-16, 8.7575e-15, 3.3197e-16, 1.4510e-17, 9.0520e-16,\n 7.7735e-17, 1.8508e-17, 8.7689e-15, 6.5441e-18, 3.0380e-17, 3.1001e-16,\n 2.3320e-16, 1.6878e-15, 1.0415e-16, 1.8601e-17, 5.0069e-16, 9.2731e-17,\n 1.5992e-15, 6.5280e-18, 2.2341e-16, 8.5357e-15, 4.6244e-15, 6.9591e-16,\n 7.0613e-18, 7.0933e-15, 2.4797e-16, 8.0887e-16, 1.1924e-16, 5.7096e-16,\n 3.5001e-16, 3.3672e-17, 1.1141e-16, 1.4765e-14, 9.3946e-16, 4.2789e-18,\n 5.3316e-17, 1.1801e-17, 6.4035e-18, 6.2548e-15, 5.1007e-17, 1.8597e-15,\n 3.8271e-15, 1.2205e-16, 6.8785e-18, 1.0908e-15, 1.5373e-15, 1.1270e-15,\n 9.9823e-17, 1.8422e-18, 1.8881e-14, 9.3390e-15, 2.5632e-15, 3.8199e-18,\n 1.8813e-15, 1.2832e-16, 1.1491e-14, 1.0854e-15, 6.3224e-16, 4.1897e-16,\n 2.7149e-15, 5.4517e-17, 4.6153e-17, 1.6039e-17, 4.5400e-17, 9.5146e-16,\n 9.1438e-15, 4.4392e-16, 4.8370e-15, 3.1387e-15, 1.3149e-15, 1.9170e-15,\n 6.4037e-17, 1.2343e-14, 8.1267e-17, 6.4826e-16, 1.8969e-17, 1.0102e-17,\n 2.0854e-17, 3.3931e-17, 8.6538e-18, 7.5309e-17, 1.4572e-16, 7.9679e-16,\n 3.3324e-15, 5.0062e-15, 9.9026e-16, 1.4789e-14, 2.1410e-15, 5.9813e-16,\n 1.6192e-15, 1.3581e-16, 2.9019e-16, 1.5269e-16, 8.6400e-15, 1.6281e-15,\n 4.2788e-16, 2.1345e-14, 3.2676e-16, 3.3699e-16, 3.6890e-17, 6.0220e-17,\n 2.5906e-14, 1.9503e-15, 1.1577e-15, 3.7274e-15, 4.0834e-16, 2.7910e-17,\n 1.6331e-15, 8.6697e-15, 1.6840e-15, 4.3949e-17, 3.0072e-17, 6.7740e-17,\n 9.9758e-16, 5.7794e-15, 5.9917e-15, 3.0244e-16, 1.3380e-15, 1.9250e-15,\n 5.0203e-17, 1.4571e-16, 4.3236e-17, 8.2354e-15, 3.3925e-17, 2.5304e-16,\n 3.5638e-17, 5.3140e-16, 5.9600e-15, 6.0062e-17, 5.1422e-16, 5.6716e-16,\n 3.5976e-17, 1.7864e-17, 1.3113e-16, 3.2246e-17, 2.9636e-17, 1.1213e-14,\n 5.1032e-15, 1.1418e-16, 1.4884e-15, 1.3626e-16, 1.0558e-14, 5.2834e-16,\n 2.4563e-17, 6.8199e-16, 9.7724e-16, 6.2034e-15, 4.7660e-17, 2.0869e-15,\n 5.0381e-16, 6.6690e-16, 4.3328e-15, 1.2401e-17, 6.4007e-18, 3.3636e-16,\n 7.8011e-17, 8.0922e-16, 7.1613e-17, 8.3625e-16, 2.4937e-16, 9.1701e-16,\n 1.0502e-16, 1.2177e-14, 1.6894e-14, 2.0079e-17, 4.2154e-16, 8.0468e-15,\n 2.4352e-15, 1.1817e-15, 4.5705e-17, 1.0321e-17, 2.4360e-16, 4.9430e-17,\n 5.1166e-17, 4.3011e-15, 7.6289e-17, 1.6197e-16, 2.0684e-15, 6.4476e-16,\n 1.9565e-17, 7.1340e-17, 7.5606e-16, 2.3474e-14, 2.1977e-17, 4.1473e-15,\n 6.0977e-16, 1.1262e-15, 1.6740e-15, 6.4838e-18, 2.6177e-17, 1.3252e-17,\n 3.4062e-15, 6.3257e-16, 8.1997e-15, 6.7363e-16, 2.8849e-15, 3.3558e-16,\n 1.8521e-14, 4.4780e-17, 1.2156e-17, 9.8756e-16, 6.2186e-17, 1.0931e-17,\n 1.1527e-16, 1.7301e-17, 2.3315e-16, 5.0497e-16, 1.2367e-14, 1.0237e-15,\n 6.8195e-15, 8.3616e-15, 1.8552e-16, 1.1217e-16, 1.6155e-15, 1.8610e-16,\n 7.0458e-18, 6.5817e-15, 1.7511e-15, 6.9421e-15, 1.1153e-16, 2.3183e-16,\n 7.0307e-16, 9.4384e-17, 1.1308e-14, 1.3276e-17, 1.9726e-15, 1.1979e-15,\n 1.1812e-17, 9.9074e-17, 3.4408e-17, 4.1523e-15, 1.2577e-15, 1.3817e-15,\n 1.3178e-16, 1.7493e-16, 1.8132e-17, 6.1192e-17, 8.4507e-15, 1.5332e-14,\n 3.7640e-15, 1.0814e-16, 1.1121e-16, 2.3505e-17], device='cuda:0')" + }, + "53": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 5.4022e-13, 2.5525e-13, 2.3031e-13, -1.5132e-14, 3.7169e-14,\n 7.8192e-14, 3.5322e-15, 9.7340e-14, 1.0613e-13, 1.6134e-14,\n 8.4457e-14, 1.4972e-13, -1.1401e-14, 4.8371e-17, 1.6454e-13,\n 2.0542e-13, 3.7534e-14, 2.9848e-15, 5.4047e-14, 8.5152e-15,\n 2.3040e-14, 2.4502e-13, -1.4739e-14, 1.3319e-13, -1.9402e-14,\n 2.1919e-13, 1.3685e-13, 8.6551e-15, 1.8488e-13, -2.4949e-15,\n 2.4631e-15, -6.4784e-15, -9.8400e-15, 3.2343e-14, -2.7529e-15,\n -4.0410e-15, 1.0140e-13, 1.0698e-13, 3.7038e-13, -1.5878e-14,\n -2.0513e-15, 2.4653e-13, 6.5491e-15, 4.5239e-14, -2.1594e-14,\n -1.4966e-14, 1.3152e-13, 7.8979e-16, 3.5055e-15, 1.8335e-14,\n 4.4962e-14, -1.2320e-14, 3.0495e-13, 1.0942e-13, 3.0775e-13,\n 1.0857e-13, 1.2523e-13, 1.0856e-13, 2.0712e-13, 6.9745e-14,\n -8.0124e-14, 1.3013e-13, 6.5397e-14, 1.5516e-13, 1.6890e-13,\n -4.6901e-16, 9.1055e-14, 7.4540e-14, 8.9883e-14, 1.6480e-14,\n 7.5782e-14, -2.2066e-14, 1.8613e-13, 2.4993e-14, 6.9404e-14,\n 6.1175e-14, 4.7814e-14, -1.0638e-14, -8.3918e-16, -1.2725e-14,\n 7.5193e-14, 1.8456e-13, -5.2083e-14, 2.3613e-13, 1.3797e-13,\n 4.6077e-14, 3.2663e-15, -3.2613e-14, -1.4443e-15, 1.9655e-14,\n 2.4581e-13, 2.3554e-13, 2.7910e-14, -3.4975e-14, 4.1858e-14,\n 1.4227e-13, -4.3293e-15, -7.2796e-15, 1.2042e-14, 9.4759e-14,\n 6.1166e-15, 2.7968e-14, -4.3854e-14, 2.3509e-13, 2.1290e-13,\n -2.7133e-15, 3.8533e-14, 1.3983e-13, 3.6890e-14, 1.4892e-14,\n -2.4470e-15, 1.9684e-13, 1.4150e-14, 1.6382e-14, 1.8961e-13,\n 3.6369e-15, -5.3168e-15, 5.7517e-14, 4.9901e-14, 6.6121e-14,\n 2.9289e-13, -8.3838e-14, 1.3377e-13, -7.3583e-15, 1.1634e-13,\n -6.0848e-14, 1.2738e-13, -2.4768e-14, -4.9916e-15, -4.5767e-15,\n 1.1541e-13, 1.3705e-15, -2.9044e-14, -7.8015e-16, 4.6984e-14,\n -3.8506e-14, 7.4652e-14, 3.7318e-16, 1.4578e-13, -4.0324e-15,\n 3.4226e-14, 1.5235e-14, -6.5944e-14, 5.5021e-14, -1.7255e-15,\n 3.6290e-13, 2.1780e-13, -1.1105e-15, -4.8483e-14, 5.4873e-14,\n 7.3193e-14, -1.9404e-14, 2.7295e-13, 3.8463e-14, 3.0013e-13,\n 6.9168e-14, 5.2065e-15, 1.7421e-13, 3.4767e-14, 1.3620e-13,\n 3.7411e-14, 1.6922e-13, 5.5328e-14, 6.5935e-14, 6.2215e-14,\n 1.8075e-13, 9.5219e-15, 5.0406e-14, -1.9909e-14, -1.0161e-15,\n 2.1629e-13, 7.1529e-14, -3.5882e-15, 2.3364e-13, 1.0042e-13,\n 8.8114e-14, 8.4202e-14, 1.0644e-14, 9.1732e-16, 1.1314e-13,\n 1.2848e-13, -2.7854e-14, 2.2583e-13, 2.1178e-13, 5.7194e-14,\n 1.2618e-13, -3.4670e-15, 9.8240e-14, 3.3567e-14, 6.3486e-14,\n -1.2020e-15, 2.7666e-13, 9.6367e-14, 1.3231e-15, 4.3142e-14,\n 3.7879e-15, 9.5774e-15, 4.9790e-14, 2.5378e-14, 1.4891e-13,\n 3.1705e-13, 1.9401e-14, 1.8188e-13, 4.4842e-14, 4.5280e-14,\n -2.7532e-13, 3.0790e-13, 1.6512e-14, 3.0233e-13, 5.9555e-14,\n 3.0998e-13, 2.4581e-13, -3.6412e-15, 3.2863e-14, 2.8206e-14,\n 3.1679e-14, 8.3781e-14, 1.8970e-13, 1.3594e-13, -2.1156e-14,\n -5.6413e-16, 1.7054e-13, 1.7240e-13, 6.4582e-14, 4.6144e-15,\n 1.6242e-13, -1.2024e-14, -4.1009e-15, -3.2824e-14, -1.4593e-14,\n 1.8694e-13, 1.0905e-13, -1.9613e-14, -1.5673e-14, -8.9446e-16,\n 6.9557e-14, 3.2607e-14, 8.7885e-16, 1.1731e-13, 9.7849e-15,\n -1.2839e-14, 3.3532e-14, 3.1995e-14, 2.5814e-14, 2.1939e-13,\n 1.1244e-13, 4.2399e-14, 1.6212e-13, 3.3420e-13, 1.5826e-13,\n 1.3785e-13, -3.7961e-14, 1.5189e-13, 2.9953e-13, -1.1482e-14,\n 1.2253e-13, 1.0765e-13, 1.9566e-13, -4.0912e-15, 4.8269e-14,\n -1.0135e-14, 7.9094e-15, 6.0389e-15, -3.6685e-15, 1.7378e-14,\n -6.1849e-16, 1.0309e-13, 5.4703e-14, -8.6045e-15, 1.2545e-14,\n 2.7915e-13, 5.0088e-14, 1.1773e-13, 1.1952e-13, 1.2944e-13,\n -2.0119e-14, 1.0886e-13, 1.2440e-14, -2.4600e-13, 9.0185e-14,\n -4.9362e-15, 1.8088e-13, -5.4744e-15, 3.5445e-15, -1.0810e-14,\n -1.9157e-14, 1.5703e-14, -1.0470e-15, -1.9223e-13, 1.7434e-13,\n 1.0358e-13, -6.7335e-15, 4.1540e-14, 6.1774e-14, -1.4258e-13,\n -4.0536e-14, 1.4142e-14, 1.3195e-15, -2.7003e-15, 1.0710e-13,\n -1.0053e-13, -1.4035e-14, -1.2278e-14, 7.9790e-14, 1.5700e-14,\n 6.8129e-14, 2.0039e-13, -1.1706e-13, 2.1205e-14, 2.2783e-13,\n -5.7402e-14, -9.8266e-15, -4.0505e-15, 3.2151e-13, 8.3588e-14,\n 3.5177e-13, 1.5334e-14, 3.0847e-15, -4.0934e-14, 4.1263e-13,\n 1.0457e-13, 2.5397e-13, 2.9389e-14, 2.6623e-14, 2.6375e-14,\n 1.1305e-13, 7.3251e-14, 7.3323e-14, 2.1376e-14, 1.9260e-13,\n 3.7882e-13, 2.6894e-13, -8.1630e-16, 6.3899e-14, -2.9794e-15,\n 2.1296e-14, 1.0079e-13, 6.6016e-14, 4.0926e-13, 1.3920e-13,\n -3.8779e-14, 1.9713e-13, 1.1470e-13, 4.1408e-14, 2.8698e-14,\n -2.9107e-14, -2.0537e-14, -3.2505e-14, -2.8741e-15, 5.3279e-14,\n 4.4262e-15, 9.8659e-14, -3.5815e-15, 1.5595e-13, 5.9134e-14,\n -1.4283e-14, 9.7969e-14, 4.2993e-15, 2.2595e-13, 1.7274e-13,\n 2.5622e-14, -7.6122e-15, -9.8231e-15, -4.5994e-14, 1.2576e-13,\n 2.9806e-13, 1.7991e-14, 6.8661e-15, -2.6991e-15, 1.1299e-13,\n 1.0897e-15, 2.7052e-13, -3.6204e-14, 1.5436e-14, -1.3242e-15,\n 4.8251e-14, 2.9989e-14, 1.8824e-13, 3.3796e-14, 3.0467e-14,\n 2.2695e-14, -3.1562e-14, 1.4357e-13, 5.0368e-14, 2.0744e-13,\n -8.9541e-14, 9.6975e-15, 6.3357e-15, 3.4831e-14, 2.5055e-13,\n 2.0174e-14, 1.6236e-13, -7.5115e-14, 1.0089e-13, 1.5791e-13,\n -4.7000e-14, -1.8875e-15, 4.0293e-14, 1.4136e-13, 6.3631e-14,\n -1.3119e-14, 2.4397e-13, -1.0916e-14, 2.6616e-14, 8.5269e-14,\n 2.9062e-14, 1.0231e-13, 9.8293e-14, 9.5048e-14, 1.9711e-13,\n 1.1288e-13, 1.6766e-14, -6.8934e-16, 9.2158e-16, -2.3470e-15,\n 2.8486e-14, 1.2322e-13, 2.3023e-13, 2.0456e-13, 1.6649e-14,\n -5.9427e-15, 1.4238e-13, 8.2123e-14, 5.8595e-14, 1.5682e-13,\n 1.6733e-13, 2.4898e-13, 7.3368e-14, 8.1651e-14, -8.4169e-14,\n 1.3541e-13, 5.6219e-16, 3.3439e-14, 4.0008e-14, 6.9622e-14,\n 1.6668e-13, 3.9486e-14, 2.0578e-13, 2.0171e-13, -5.0285e-15,\n 1.5998e-15, 7.3275e-14, 6.4660e-14, 8.2375e-14, -2.3044e-14,\n -6.2478e-16, -6.7190e-14, 4.0886e-14, 5.9639e-14, -1.1193e-15,\n 7.5220e-14, 2.8971e-13, 7.3940e-14, 8.5646e-14, 4.2212e-14,\n 2.1736e-14, 1.2671e-13, -7.7737e-16, 5.1796e-14, 1.0574e-13,\n 9.9003e-14, 4.9133e-14, 1.3965e-13, -5.7028e-15, 2.5128e-13,\n 5.1001e-16, 2.5384e-14, 1.9864e-13, -3.9588e-14, 4.5571e-14,\n 3.7772e-15, 1.7691e-13, 3.6741e-14, 3.3603e-14, 1.1363e-13,\n -6.6016e-15, 1.1933e-15, 1.1663e-13, 2.6773e-14, 6.7571e-14,\n 1.8635e-13, -9.0113e-14, 1.6562e-13, -5.7505e-14, 2.9679e-13,\n 3.0874e-14, 1.0851e-13, 1.1023e-13, 7.1866e-15, 1.2093e-13,\n 3.6995e-13, 2.0501e-14, 1.1367e-16, 3.0443e-13, 1.0839e-13,\n 2.8549e-13, 1.9448e-15, -1.0013e-15, -1.5068e-13, 1.3081e-13,\n 1.6289e-14, 1.0455e-13, 2.2274e-13, 2.7012e-15, 2.8291e-14,\n 6.5869e-14, 7.1166e-14, -1.5429e-15, -2.0685e-14, 1.1966e-13,\n 4.0523e-14, 1.4285e-13, 1.5226e-13, -4.9684e-14, 1.7925e-13,\n 7.7562e-14, 1.5619e-13, -8.1862e-14, 1.2511e-13, 2.4244e-13,\n 2.2977e-13, 1.0128e-13, -4.1922e-15, 6.7354e-14, 1.4926e-13,\n 9.1324e-15, 3.9609e-16, 1.3405e-13, 3.6674e-15, -8.2606e-15,\n 5.7174e-14, 2.7934e-14, -8.2418e-15, 4.0325e-13, 2.0199e-13,\n 2.9227e-13, 1.9535e-14, -1.0749e-14, 4.9801e-14, 7.2029e-14,\n 2.2171e-14, 5.4602e-14, 1.4578e-14, 2.3265e-13, 4.4491e-15,\n 1.0041e-13, 1.8506e-13, 1.7327e-14, 8.1826e-14, 1.6406e-15,\n 4.1224e-14, -4.2709e-14, -6.0881e-15, 3.9594e-14, 2.9504e-15,\n 8.0257e-14, 4.6621e-15, 1.6326e-13, 6.3587e-14, 1.7690e-13,\n 2.8529e-13, 1.5017e-13, 6.5876e-14, 1.3127e-13, -6.4501e-15,\n 3.2047e-14, 1.0444e-13, -4.1275e-15, 4.6956e-14, -2.2777e-14,\n 1.8889e-14, 2.3824e-13, 1.6467e-13, 2.0716e-13, 7.5620e-14,\n 1.4728e-15, 2.4548e-15, 2.3117e-13, 1.4225e-13, 2.0945e-13,\n 1.2650e-15, 1.6755e-14, 8.3555e-14, 1.6409e-13, 3.1435e-13,\n 5.7102e-14, 3.0878e-15, 1.4644e-13, -2.5664e-14, 3.2394e-14,\n 6.3509e-14, 1.7250e-13, 1.6122e-13, 3.7869e-14, 3.9020e-14,\n 3.7454e-13, 2.8186e-15, -5.3322e-15, 5.8672e-14, 6.2713e-14,\n 1.3605e-13, 1.6550e-13, 1.8175e-13, 1.3053e-13, 5.4518e-15,\n 2.5837e-13, -1.7341e-13, 2.5278e-13, -4.2232e-15, -6.5236e-15,\n 3.2048e-14, 9.7972e-15, 8.5495e-15, 3.1236e-13, -2.8359e-14,\n 2.7788e-14, 2.1993e-15, -1.1950e-14, 7.9026e-14, 1.3552e-13,\n -1.4789e-14, 1.4109e-13, 1.3273e-13, 3.5570e-13, 1.8127e-14,\n -2.8902e-14, 1.7509e-15, 1.3983e-14, 2.3311e-15, 1.9407e-13,\n 6.1512e-14, 1.9296e-13, 4.9261e-15, 6.5462e-15, 6.8863e-14,\n 1.9787e-13, 9.5940e-14, 4.8070e-15, 2.2336e-15, 7.1784e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([4.4539e-15, 1.1184e-16, 1.4862e-15, 5.1487e-17, 2.0944e-15, 1.2739e-15,\n 9.6142e-17, 3.3115e-17, 4.5535e-15, 1.0568e-15, 2.0062e-15, 2.3974e-15,\n 1.7608e-15, 5.3635e-18, 1.5205e-15, 9.6338e-16, 3.8162e-15, 2.8883e-15,\n 5.2514e-16, 9.0673e-16, 6.1975e-16, 1.1642e-15, 1.5067e-15, 2.1634e-15,\n 1.0723e-15, 2.0915e-15, 1.4847e-15, 1.7480e-16, 8.2285e-15, 8.1586e-16,\n 1.2561e-17, 1.2199e-15, 2.3834e-17, 2.2504e-16, 4.9787e-18, 7.3337e-16,\n 2.2654e-15, 5.4184e-15, 8.5420e-15, 5.6609e-17, 9.8499e-16, 3.5165e-16,\n 3.8321e-16, 3.6622e-16, 1.5318e-17, 1.9674e-16, 6.5590e-15, 2.3482e-16,\n 4.8682e-16, 6.1393e-16, 6.8282e-17, 1.4693e-17, 2.9364e-16, 7.8189e-17,\n 2.9903e-15, 3.4015e-16, 1.5271e-16, 2.1798e-15, 6.0893e-15, 7.1226e-16,\n 1.0464e-15, 8.0520e-15, 3.9331e-15, 7.2014e-15, 5.6112e-16, 7.5799e-18,\n 2.9511e-15, 4.2653e-15, 2.7808e-15, 1.0473e-15, 1.7493e-15, 4.8934e-18,\n 2.3924e-15, 3.1168e-15, 1.5973e-15, 3.9301e-15, 6.7903e-16, 5.7398e-18,\n 2.0978e-17, 4.0790e-16, 1.3996e-15, 2.0892e-15, 2.7516e-16, 9.5283e-16,\n 4.3365e-15, 3.7421e-16, 1.9242e-16, 5.9898e-16, 1.2892e-17, 2.7646e-15,\n 1.8467e-15, 2.5409e-15, 5.8454e-16, 1.7708e-15, 4.1637e-15, 1.1138e-15,\n 1.6513e-16, 1.2798e-15, 4.2324e-16, 2.7128e-16, 2.1970e-15, 1.2473e-15,\n 9.3782e-16, 1.3155e-15, 3.1039e-15, 1.4434e-15, 3.8525e-15, 1.5124e-17,\n 1.8691e-17, 1.6030e-16, 1.6560e-17, 1.2034e-15, 6.6011e-16, 3.0076e-15,\n 2.1409e-16, 4.1724e-17, 2.2125e-17, 2.4456e-16, 2.9056e-15, 4.6911e-15,\n 5.5203e-15, 4.8411e-17, 4.4776e-15, 2.2417e-15, 1.3775e-16, 2.8868e-15,\n 8.5902e-16, 1.7072e-15, 4.4265e-16, 2.1352e-17, 1.8666e-16, 5.3592e-16,\n 6.0410e-17, 3.0115e-17, 4.4149e-15, 1.5077e-15, 4.7284e-15, 7.9910e-16,\n 1.2041e-15, 2.1607e-16, 1.1264e-15, 8.0784e-16, 5.4996e-15, 7.6392e-16,\n 7.5216e-17, 6.0254e-15, 3.3647e-15, 1.7563e-15, 5.1026e-17, 2.4962e-15,\n 1.9442e-15, 2.5239e-16, 2.7679e-15, 2.3269e-15, 1.2101e-15, 2.2881e-15,\n 2.2218e-15, 1.3545e-15, 4.0712e-17, 1.9334e-16, 8.1444e-16, 2.6900e-16,\n 1.9369e-17, 5.8128e-15, 1.1391e-15, 9.3919e-17, 3.2210e-16, 4.2316e-17,\n 1.1831e-15, 9.7622e-18, 5.2101e-15, 2.2674e-15, 1.7795e-16, 1.7979e-15,\n 2.1167e-15, 5.9981e-16, 2.3253e-15, 2.0796e-16, 1.0578e-15, 3.8893e-15,\n 9.7685e-16, 1.7222e-16, 5.2527e-15, 1.1167e-15, 7.6020e-15, 6.6817e-15,\n 8.2178e-18, 7.2762e-16, 4.0748e-16, 2.7504e-15, 1.8325e-15, 6.0041e-15,\n 2.6646e-16, 5.2108e-17, 2.1335e-15, 1.0876e-17, 3.2080e-16, 3.2479e-15,\n 1.1916e-14, 8.8353e-16, 1.5682e-15, 7.2015e-15, 3.0571e-15, 1.0411e-15,\n 3.2755e-17, 2.8645e-15, 3.3210e-15, 3.0415e-15, 4.0639e-15, 2.3983e-15,\n 1.5774e-15, 3.1649e-15, 1.0268e-17, 1.0829e-15, 3.4217e-17, 5.7368e-16,\n 2.3432e-16, 9.7626e-16, 2.1297e-15, 2.1822e-16, 7.2429e-18, 1.3989e-15,\n 1.9107e-15, 5.4011e-15, 1.9259e-15, 4.0400e-17, 2.1450e-16, 8.0167e-16,\n 1.3456e-16, 2.1147e-16, 3.1409e-15, 1.4585e-15, 1.4266e-15, 2.4551e-16,\n 1.1228e-17, 2.8584e-15, 2.1372e-15, 6.4665e-18, 5.9539e-16, 2.6173e-16,\n 1.9428e-15, 2.0324e-15, 5.4705e-15, 2.4858e-15, 6.1897e-16, 3.1138e-16,\n 1.1230e-15, 2.2762e-15, 4.6940e-15, 9.2232e-16, 7.7479e-16, 2.4014e-15,\n 9.3515e-16, 5.0534e-17, 4.5725e-17, 3.0255e-16, 6.0138e-15, 1.0598e-14,\n 2.5197e-17, 2.5818e-16, 1.0326e-15, 1.6544e-15, 1.8632e-17, 1.0097e-15,\n 1.0978e-15, 1.9191e-15, 2.0535e-16, 5.1748e-16, 2.7002e-15, 2.1595e-17,\n 5.9949e-16, 2.4426e-16, 5.6124e-15, 1.2807e-15, 1.9340e-15, 2.7789e-17,\n 3.6539e-15, 1.0129e-15, 1.3890e-15, 2.9676e-15, 1.1409e-17, 3.1454e-15,\n 8.1381e-17, 4.0016e-15, 1.3519e-16, 1.8695e-17, 1.1860e-15, 1.0175e-15,\n 6.6614e-15, 2.4027e-15, 5.6163e-15, 1.2252e-15, 6.3929e-16, 2.2656e-16,\n 2.0571e-15, 5.0091e-15, 5.9457e-16, 5.3104e-17, 1.6682e-15, 4.9030e-15,\n 1.0998e-15, 3.5002e-16, 1.7858e-15, 1.0241e-14, 2.2167e-15, 2.1851e-17,\n 1.2473e-14, 1.5315e-14, 3.1958e-15, 4.6616e-15, 5.5061e-15, 1.9052e-15,\n 1.5774e-17, 8.2647e-15, 4.5684e-15, 1.6399e-16, 3.3657e-17, 7.7383e-16,\n 1.1428e-15, 5.3364e-17, 2.2757e-15, 8.3946e-16, 1.5253e-15, 2.1840e-15,\n 4.3155e-16, 3.7743e-15, 2.9162e-17, 4.3045e-15, 6.1892e-15, 4.1591e-15,\n 5.4212e-15, 2.4506e-15, 7.0583e-16, 3.1670e-17, 8.6616e-15, 8.2714e-16,\n 3.5212e-15, 2.4529e-15, 1.4618e-15, 3.5428e-15, 5.1735e-18, 1.5894e-15,\n 3.3939e-15, 5.3024e-16, 8.0476e-16, 2.5282e-15, 3.3292e-16, 2.1047e-17,\n 1.0844e-17, 1.4507e-16, 3.7298e-16, 3.4027e-15, 2.2991e-15, 6.5876e-16,\n 4.0169e-15, 5.5439e-17, 2.2867e-15, 2.8068e-17, 3.4494e-16, 1.5364e-15,\n 3.9752e-15, 1.3945e-15, 6.3186e-16, 2.3035e-15, 1.9272e-15, 2.5567e-17,\n 2.1892e-15, 2.5290e-16, 6.6591e-17, 7.1115e-16, 5.7099e-15, 4.2214e-16,\n 1.6690e-16, 8.7183e-16, 4.0389e-17, 7.2941e-17, 8.1308e-16, 4.1526e-15,\n 1.5050e-15, 3.8057e-15, 9.6447e-16, 2.8403e-15, 2.7931e-15, 9.9118e-16,\n 2.5116e-15, 8.0313e-15, 1.5105e-15, 7.8907e-16, 1.1232e-15, 1.9546e-15,\n 5.5344e-16, 1.3294e-15, 4.6522e-16, 7.1141e-17, 4.1653e-15, 1.8124e-17,\n 1.1275e-17, 2.3914e-15, 2.7310e-15, 8.1699e-16, 3.6444e-16, 1.8471e-15,\n 1.7104e-17, 6.1955e-16, 5.6666e-15, 2.1130e-16, 5.5879e-16, 1.2976e-15,\n 1.5112e-15, 2.6889e-15, 4.9698e-16, 3.8389e-16, 3.8443e-16, 2.2722e-17,\n 1.2770e-15, 5.6345e-16, 1.5928e-15, 9.0875e-15, 7.6702e-15, 1.5720e-15,\n 7.2251e-17, 4.8198e-15, 2.2955e-15, 1.0112e-15, 1.2928e-15, 8.5116e-16,\n 9.7952e-16, 9.6153e-16, 4.2297e-17, 1.0743e-14, 2.2861e-15, 6.3184e-17,\n 6.6329e-16, 2.6910e-16, 2.0563e-17, 4.5371e-15, 1.3919e-15, 2.0013e-15,\n 2.7011e-15, 4.8050e-17, 2.2827e-16, 2.2208e-15, 3.8786e-15, 2.0233e-15,\n 1.0822e-16, 2.1065e-16, 7.8500e-15, 7.6612e-15, 4.6974e-15, 4.2673e-17,\n 3.0466e-15, 1.1466e-15, 5.4923e-15, 1.6878e-15, 2.5052e-15, 1.3346e-15,\n 3.1464e-15, 1.2199e-17, 4.3958e-16, 3.9959e-16, 1.7693e-15, 2.7640e-15,\n 4.8155e-15, 1.2735e-15, 3.3617e-15, 4.2089e-15, 2.3935e-15, 2.3764e-15,\n 2.6531e-16, 4.8179e-15, 7.8000e-16, 9.6535e-16, 7.9762e-16, 2.8374e-16,\n 1.4995e-15, 7.1040e-18, 4.4140e-16, 3.9744e-16, 1.3743e-15, 2.2909e-15,\n 2.3537e-15, 4.1008e-15, 1.6902e-15, 6.3981e-15, 2.0096e-15, 1.4802e-15,\n 5.2659e-15, 5.1528e-16, 8.6420e-16, 9.1672e-16, 8.7715e-15, 5.5118e-15,\n 9.2848e-16, 4.0560e-15, 1.6078e-15, 8.7124e-16, 6.8797e-16, 1.2211e-17,\n 5.5606e-15, 2.0739e-15, 1.7911e-15, 2.4044e-15, 1.1774e-15, 1.2266e-16,\n 2.2799e-15, 4.2828e-15, 3.3359e-15, 1.0984e-17, 1.4004e-17, 3.1935e-17,\n 3.0506e-15, 2.8582e-15, 4.0846e-15, 2.3575e-15, 2.8381e-15, 6.1747e-15,\n 1.0024e-15, 1.3493e-15, 1.0875e-15, 4.7149e-15, 7.9861e-16, 5.9784e-16,\n 1.5910e-16, 1.0659e-15, 3.1612e-15, 3.6513e-17, 1.1400e-15, 2.7605e-15,\n 8.0723e-18, 2.0089e-16, 5.9175e-16, 2.0936e-17, 2.3253e-16, 8.3989e-15,\n 5.2968e-15, 8.4671e-16, 2.2838e-15, 5.8216e-17, 3.2981e-15, 1.2987e-15,\n 8.8506e-16, 1.3964e-15, 1.8511e-15, 1.3657e-15, 3.0000e-17, 1.5705e-15,\n 1.8045e-15, 1.4897e-15, 2.3074e-15, 4.0931e-16, 4.7158e-17, 8.0194e-16,\n 4.1890e-16, 1.8145e-15, 1.0381e-15, 1.8229e-15, 1.4999e-15, 2.7813e-15,\n 8.3211e-16, 8.1590e-15, 1.4313e-14, 5.1771e-16, 8.2382e-16, 4.1558e-15,\n 2.8299e-15, 2.1935e-15, 5.7210e-16, 1.4949e-17, 2.2096e-15, 1.3861e-17,\n 3.1370e-16, 3.0071e-15, 5.8208e-16, 1.4735e-15, 2.4813e-15, 1.3091e-15,\n 1.7468e-16, 1.8554e-16, 1.0077e-15, 5.5271e-15, 2.0604e-16, 2.8611e-15,\n 3.9940e-15, 2.1928e-15, 1.3910e-15, 5.0133e-16, 6.0923e-16, 8.0734e-16,\n 4.3764e-15, 1.4384e-15, 3.7831e-15, 2.9116e-15, 3.0917e-15, 5.5535e-16,\n 5.5809e-15, 1.0542e-15, 3.5305e-16, 2.0161e-15, 3.3984e-17, 1.4843e-16,\n 5.0422e-16, 2.8885e-16, 8.0003e-16, 1.2070e-15, 1.5885e-14, 1.2806e-15,\n 4.3048e-15, 2.5595e-15, 1.6236e-15, 4.0709e-17, 2.0218e-15, 1.1060e-15,\n 1.3445e-15, 4.3265e-15, 3.2833e-15, 3.6426e-15, 2.9577e-17, 5.8239e-16,\n 3.0513e-15, 2.3345e-16, 3.0553e-15, 3.6262e-17, 2.1136e-15, 4.1444e-15,\n 9.4705e-17, 8.0782e-16, 6.3569e-16, 3.8669e-15, 7.3410e-15, 2.9247e-15,\n 1.3453e-15, 7.6447e-16, 6.0037e-18, 4.3554e-17, 5.8405e-15, 1.2804e-14,\n 2.4190e-15, 7.8877e-16, 2.9937e-17, 3.5626e-16], device='cuda:0')" + }, + "54": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 5.6052e-45, 1.7505e-16, -1.0031e-17, ..., -3.2413e-16,\n -2.1501e-15, 7.8272e-18],\n [ 5.6052e-45, 9.7517e-17, 1.4743e-17, ..., -2.9670e-16,\n 1.1456e-15, 2.3640e-18],\n [-5.6052e-45, -5.3336e-16, 6.6377e-17, ..., 4.7688e-15,\n -9.9701e-15, 9.5430e-17],\n ...,\n [-5.6052e-45, -5.4381e-16, -8.5164e-17, ..., -3.9955e-15,\n -1.2141e-16, -1.2413e-16],\n [ 5.6052e-45, 3.4270e-16, -1.1831e-17, ..., -1.5455e-15,\n -2.7828e-15, -1.7108e-17],\n [ 5.6052e-45, -7.6208e-16, -9.2232e-19, ..., 2.8528e-16,\n -2.4081e-15, -3.7432e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.4778e-16, 3.9286e-16, 4.6929e-16, ..., 3.3137e-19, 1.7595e-17,\n 3.0592e-16],\n [1.9035e-17, 5.6639e-17, 6.5839e-17, ..., 1.5460e-19, 2.6475e-18,\n 3.3695e-17],\n [1.3169e-15, 2.6726e-15, 3.2809e-15, ..., 1.4780e-17, 2.1793e-16,\n 1.7327e-15],\n ...,\n [5.3572e-17, 4.8517e-17, 1.2009e-16, ..., 6.7570e-18, 1.8008e-17,\n 3.2488e-17],\n [8.7757e-16, 2.4790e-15, 2.9594e-15, ..., 4.6924e-18, 2.1187e-16,\n 1.5743e-15],\n [2.1792e-15, 9.5760e-15, 1.5312e-14, ..., 2.8240e-17, 2.9893e-16,\n 6.1919e-15]], device='cuda:0')" + }, + "55": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-6.0212e-13, 7.2842e-13, -1.1068e-12, 5.3854e-12, 1.6579e-12,\n -1.9813e-12, -3.3867e-12, 5.4395e-13, 1.1193e-12, -1.4222e-12,\n 1.0422e-11, -4.2690e-12, -2.4871e-13, -2.7002e-12, 9.8224e-12,\n 2.1580e-12, 1.0058e-12, -1.3427e-12, 3.4153e-12, -2.3756e-12,\n -7.2463e-13, 1.7798e-12, -4.3145e-12, -2.0378e-12, 2.1046e-12,\n 2.2900e-12, 2.8120e-12, -7.5465e-12, 3.5441e-12, -9.4979e-13,\n 1.0169e-12, 2.4085e-12, -1.1741e-12, 5.5478e-12, 4.4837e-13,\n -3.8484e-12, -1.7239e-13, 2.9375e-13, 2.2537e-12, 1.2942e-12,\n -3.0354e-13, -1.2859e-12, -1.2619e-12, -2.5095e-12, -1.2220e-12,\n 5.8528e-12, 6.4919e-13, 2.9949e-12, -1.7458e-13, -8.5676e-13,\n -8.6628e-13, -2.6924e-12, -1.8045e-12, -1.8078e-12, -2.5134e-13,\n -9.9881e-13, 1.7635e-12, 2.6703e-12, -8.8660e-13, -1.4919e-12,\n -1.3834e-12, -1.4000e-12, 3.3889e-12, 4.6100e-13, 1.2780e-12,\n -7.2071e-13, -1.3663e-12, 3.7802e-12, -1.3150e-12, 4.6327e-12,\n 8.5673e-12, 4.5440e-12, -8.9134e-13, -1.3682e-12, 4.0868e-14,\n -2.6750e-12, -8.3872e-13, 1.3325e-13, 3.4575e-12, -1.4989e-12,\n -7.5294e-13, -1.9496e-13, -2.1436e-12, 2.6949e-12, -2.3000e-12,\n 8.9878e-13, -7.4551e-13, -3.8402e-13, -2.3607e-12, -1.2844e-12,\n 3.8044e-13, 2.0288e-12, -7.4783e-12, -4.6293e-12, -6.1527e-13,\n -1.3045e-12, -1.6094e-12, -1.9897e-12, -5.4492e-12, 1.3233e-12,\n -6.5396e-13, -1.4337e-12, -1.8981e-13, 4.0606e-12, -2.6623e-13,\n -4.0782e-12, 3.1085e-13, 1.7767e-12, 7.2813e-12, -2.0080e-12,\n -4.1538e-12, -1.4542e-12, -1.1505e-12, -2.3876e-13, 1.9106e-12,\n 1.9202e-12, -1.3939e-15, -1.2833e-12, -2.2684e-12, -8.9535e-13,\n -8.0716e-13, -3.9100e-12, 5.0674e-12, 2.9536e-12, 7.7050e-12,\n 4.7270e-12, 8.1650e-13, -9.4953e-12, 5.6937e-13, 4.4541e-13,\n -1.1541e-12, -2.8711e-12, -1.3208e-12, -2.7128e-12, 2.3172e-12,\n 4.4125e-12, 4.9169e-12, -6.5291e-12, 1.1867e-11, -1.4349e-13,\n 1.9447e-12, -1.4123e-12, -6.6923e-13, 3.3001e-12, 4.6059e-13,\n 5.3705e-12, 3.3777e-13, -1.7248e-12, -6.7418e-13, -2.7976e-12,\n -1.5060e-12, 3.1533e-12, -1.4499e-12, 1.5782e-12, 4.4306e-12,\n 5.7162e-12, 3.0608e-12, 3.8528e-12, -3.9148e-12, 3.0936e-12,\n 2.2526e-12, 7.4793e-13, -5.3926e-14, -2.8624e-13, -1.3345e-12,\n 7.4584e-12, -9.7636e-13, -2.9173e-12, -1.3736e-12, -7.8835e-13,\n -4.9701e-13, 1.4722e-12, -2.6623e-12, -1.2654e-12, -6.3586e-13,\n -3.6298e-12, -1.0455e-12, -3.3767e-12, -5.9145e-13, -1.1212e-12,\n 3.2232e-12, 5.1276e-12, -1.2346e-13, 7.3121e-12, -2.8594e-12,\n -1.3276e-12, -1.3522e-12, -7.6992e-12, -2.2090e-12, -2.0452e-13,\n 9.3549e-13, 1.5889e-12, 3.2023e-13, -3.1485e-12, 6.0894e-13,\n -2.8589e-12, -6.1439e-13, 1.5080e-12, -1.6596e-12, -5.1012e-13,\n 1.4553e-12, -4.7296e-12, 4.3535e-13, -3.0537e-12, 1.9744e-12,\n -7.5254e-12, 3.8779e-12, -5.7917e-12, 9.4268e-12, -9.6487e-13,\n 1.3215e-12, 5.4120e-12, 1.2552e-12, -2.7162e-13, 1.8531e-12,\n -7.4450e-13, -1.3127e-12, 2.9921e-12, -1.3687e-13, 7.3612e-13,\n -4.6908e-12, 5.0201e-12, -1.4073e-12, -2.1813e-12, -1.4454e-12,\n 2.9904e-12, -1.2674e-12, -2.0786e-12, 7.3269e-12, -1.1198e-12,\n 9.2627e-13, 4.9242e-12, -9.4757e-13, 2.4773e-12, -2.2079e-12,\n -2.5176e-12, -4.6429e-12, -5.0678e-12, -9.5763e-13, 1.7944e-13,\n 8.7800e-12, -1.2672e-12, -5.1486e-12, 2.1429e-12, 1.4320e-13,\n -1.2235e-12, 5.0812e-12, 2.3191e-13, -6.0398e-13, 4.6226e-12,\n -1.0608e-12, -9.3032e-12, 2.3995e-12, 4.8752e-13, -1.6236e-12,\n -1.8570e-12, -1.9806e-12, 1.1060e-12, -8.6067e-13, -1.6872e-12,\n -1.3144e-12, -1.3053e-12, 2.1286e-12, 4.4989e-12, -6.5802e-13,\n -2.0957e-12, -4.2430e-12, 7.9487e-12, 6.6220e-13, 7.6462e-12,\n 5.1014e-12, 1.8166e-12, -1.6333e-12, 1.9277e-12, -1.0795e-12,\n -1.1296e-12, -7.1957e-12, 2.2852e-13, -5.8282e-12, 9.0943e-13,\n -3.3437e-12, -1.4055e-12, -7.8899e-13, -2.2364e-12, 5.8626e-13,\n 4.5208e-14, -4.4657e-12, -1.6359e-12, -1.2163e-11, 1.6894e-13,\n -1.5969e-12, -1.8223e-12, -1.4624e-12, 3.2576e-12, -4.3707e-12,\n 1.9553e-12, -8.4521e-13, -4.3846e-12, -7.4242e-12, 1.6354e-13,\n -4.2640e-12, -3.7510e-12, -5.0762e-12, -9.0986e-14, -8.9701e-14,\n 2.0019e-12, -1.6925e-12, 1.6042e-12, 1.5473e-12, 4.7029e-12,\n -4.3964e-12, 3.5959e-12, -1.6588e-13, 9.9084e-12, -3.6559e-12,\n 4.6522e-12, -1.4984e-12, 6.1659e-12, -7.0846e-12, -4.7324e-13,\n -1.5790e-12, 3.4230e-12, -2.8187e-12, -9.3087e-13, 4.6397e-12,\n 5.0378e-12, 7.9213e-12, 1.0889e-11, -2.7085e-12, -2.7618e-12,\n 8.4410e-12, 4.9747e-12, -1.6886e-12, -1.5240e-12, 1.2185e-11,\n -1.5970e-12, 3.1413e-12, -7.1935e-13, -1.1797e-12, 4.0358e-13,\n -6.2519e-14, 4.7971e-12, 5.5276e-13, -1.8008e-12, -3.2302e-12,\n -1.6508e-12, 1.5281e-12, 5.7012e-12, -1.6324e-12, 2.9113e-13,\n -3.8138e-13, 3.6659e-12, -5.5198e-13, -1.0624e-12, 1.4521e-12,\n -1.5924e-12, 2.6109e-12, -1.1424e-12, 3.5858e-12, -1.3848e-12,\n 1.9286e-12, 5.4096e-12, -2.2146e-12, -7.5523e-12, 7.3673e-12,\n 5.3558e-12, 2.1950e-13, -3.2132e-12, -4.1736e-12, -1.0160e-13,\n -1.2951e-12, 1.0770e-11, 4.3844e-12, 1.8770e-12, -9.6080e-13,\n -1.1466e-13, -1.7037e-12, -2.0759e-12, -1.1186e-12, -1.1541e-12,\n -1.0737e-12, 1.1026e-12, 4.5554e-12, -3.2417e-12, 9.4308e-14,\n -3.5467e-12, -1.7396e-12, 4.1109e-12, -1.3103e-12, -5.3505e-13,\n 2.7274e-12, -6.2118e-13, -5.8794e-12, -1.4069e-12, 4.0291e-12,\n -5.5004e-12, 2.3839e-12, -2.5608e-12, 3.7883e-12, -9.8648e-13,\n -3.8847e-13, 9.6096e-14, -1.3534e-12, -3.5889e-12, 9.1385e-12,\n -1.1404e-12, -1.3632e-12, -1.7206e-12, -1.1571e-12, 3.3895e-13,\n 4.8893e-12, 7.9407e-12, -7.3909e-13, -2.8866e-12, -1.3751e-12,\n -1.6903e-12, 1.0419e-11, -1.2869e-12, 2.3142e-12, -2.1575e-12,\n -1.0164e-12, -1.3536e-12, -1.6056e-12, -8.4008e-13, -1.2262e-12,\n 1.6514e-12, -1.8089e-13, 5.9340e-12, -1.1104e-12, -1.0421e-11,\n -3.0538e-12, 4.7230e-12, 1.7520e-12, 3.9417e-12, -3.0366e-12,\n 3.5327e-12, -1.1942e-12, 6.9126e-13, 8.2367e-13, -7.0573e-12,\n -1.5326e-12, -1.9110e-12, -1.1271e-12, -3.2187e-12, -1.7090e-12,\n -1.4952e-12, -3.0930e-12, -9.6238e-13, -2.1901e-12, -8.1279e-13,\n -1.3038e-12, -1.1616e-12, 1.6175e-12, -1.1900e-12, -1.0864e-12,\n -1.8181e-12, -3.2961e-13, 5.1164e-13, 1.8165e-12, -5.9958e-13,\n -1.4786e-12, 3.4282e-12, 7.5375e-13, 7.6609e-13, -1.6820e-12,\n -2.8875e-12, -1.2358e-12, 2.4194e-12, -2.7576e-12, -1.5872e-12,\n -5.2368e-12, 2.2491e-13, -1.1131e-12, -1.4132e-12, 4.2821e-13,\n -1.1445e-12, -9.0748e-12, -4.6433e-13, -1.9345e-12, -1.1157e-12,\n 2.6655e-12, -5.4353e-12, -1.7846e-12, -3.2670e-12, 1.2487e-11,\n -2.0633e-12, 8.4527e-12, 5.1664e-12, 9.4307e-12, -9.8797e-13,\n -1.8120e-12, -3.6975e-13, -3.8253e-12, -4.4409e-14, -1.4872e-12,\n -6.3764e-13, -4.3037e-12, -1.2355e-12, -5.8232e-12, 1.2145e-12,\n -3.5363e-12, 1.0843e-12, 4.6724e-12, -1.1899e-12, -1.6933e-12,\n -5.6364e-13, -1.5758e-12, -1.3180e-12, -1.1887e-12, 1.4829e-12,\n -4.8240e-13, 1.4959e-12, -2.3906e-12, -2.4274e-12, -3.7666e-12,\n 1.5873e-12, -1.3561e-12, -7.2410e-12, 3.8654e-12, -5.1388e-14,\n 3.0193e-12, 2.1347e-13, -1.1256e-12, 1.2626e-12, -1.3034e-12,\n -9.0181e-13, -4.2530e-12, -1.4675e-12, -3.3117e-12, -1.3110e-12,\n -1.6834e-12, 6.6908e-13, 1.6389e-12, 1.3988e-11, 2.5571e-12,\n -1.2781e-12, -1.2610e-12, 2.4191e-12, -9.4845e-13, -1.3630e-12,\n -7.2594e-13, 1.3980e-12, -2.6085e-12, 1.2590e-12, -1.1524e-12,\n 2.3855e-12, -1.3850e-12, -1.1335e-12, -1.0228e-12, -1.2596e-12,\n 1.3627e-13, 3.2239e-13, -1.3396e-12, -3.8810e-12, -6.7719e-12,\n 1.0281e-12, -1.7297e-13, 6.1766e-13, 9.1288e-13, 7.3444e-13,\n 1.2173e-11, 1.6205e-13, -1.4779e-12, -2.2283e-12, -3.5784e-12,\n -1.0889e-12, -1.1528e-12, -1.8504e-12, -1.1958e-12, 1.3469e-12,\n 3.4089e-12, -3.2782e-13, -3.3289e-13, 4.8437e-12, -5.3602e-12,\n -1.3180e-12, -1.8024e-12, -3.7016e-13, -1.5200e-12, 2.0422e-12,\n -3.9725e-13, 1.1266e-12, 8.9194e-14, -1.2439e-12, 1.1924e-11,\n -8.5233e-13, -1.8146e-12, -1.4892e-12, -4.5309e-12, -1.4572e-12,\n -8.0113e-13, -1.4325e-12, 1.1805e-12, -4.9599e-13, -1.1414e-12,\n 7.4733e-12, -9.6493e-14, 3.6947e-13, 1.2804e-12, -7.3587e-16,\n 2.4009e-12, -6.3964e-14, 6.8870e-13, 6.2050e-12, -4.1558e-12,\n -1.3720e-12, -4.2911e-12, -7.6940e-14, -2.2258e-12, -8.8242e-13,\n -1.2382e-12, -2.9506e-12, -4.3765e-13, 1.8005e-12, -2.2536e-12,\n 4.9674e-13, -1.4477e-12, -9.9754e-13, -3.5689e-12, 1.2257e-12,\n -2.6930e-12, 3.2302e-12, 4.4294e-12, -5.8524e-13, 7.5552e-13,\n -4.6139e-12, 3.5980e-12, -1.0772e-12, 4.9676e-12, 4.3788e-12,\n -1.0353e-12, -1.0847e-12, -3.3495e-12, 7.6605e-12, -7.2243e-13,\n 7.1268e-13, -7.0660e-13, -2.9804e-12, -4.6905e-12, 2.0787e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.5931e-13, 9.5431e-14, 2.5219e-12, 2.3437e-12, 5.6167e-13, 3.9008e-12,\n 7.0953e-13, 3.4899e-13, 1.7186e-12, 1.1529e-12, 2.4823e-13, 4.7530e-12,\n 2.5627e-12, 5.1769e-13, 6.1248e-13, 2.1131e-12, 2.5321e-12, 1.6523e-12,\n 2.6656e-13, 4.0231e-13, 5.3299e-13, 1.6891e-13, 6.2032e-12, 7.2840e-12,\n 5.8309e-12, 5.3263e-12, 7.2607e-12, 3.0514e-12, 5.0574e-12, 1.8116e-12,\n 1.9984e-12, 1.0467e-12, 4.7611e-12, 3.4323e-12, 2.9405e-13, 1.6514e-12,\n 5.7850e-14, 5.3330e-13, 1.3448e-11, 2.4168e-13, 1.5068e-13, 1.7339e-12,\n 7.0907e-13, 8.9738e-14, 4.6071e-13, 3.9328e-12, 1.7608e-11, 3.1323e-12,\n 2.4396e-13, 9.8311e-13, 1.4090e-12, 1.3513e-13, 1.0443e-13, 1.6773e-12,\n 3.4089e-13, 1.2611e-13, 2.5674e-12, 1.3797e-12, 1.7707e-13, 6.6888e-14,\n 9.7953e-13, 1.2302e-13, 3.0533e-13, 5.2033e-12, 2.5081e-12, 2.0590e-12,\n 6.8251e-14, 6.1480e-12, 5.9050e-13, 3.8296e-12, 4.3940e-12, 1.3750e-12,\n 5.6565e-12, 3.9412e-12, 3.3106e-12, 1.2844e-12, 3.6254e-13, 6.7634e-13,\n 1.7383e-13, 3.2343e-12, 2.3964e-12, 1.5022e-12, 4.2891e-13, 3.2748e-12,\n 4.3777e-13, 3.0680e-12, 2.6311e-12, 9.0933e-14, 1.7310e-12, 4.3236e-13,\n 2.7049e-12, 1.8174e-12, 6.1141e-12, 6.2710e-13, 1.1976e-12, 4.1288e-13,\n 1.7591e-12, 9.6268e-13, 2.1536e-12, 2.5673e-12, 1.9051e-12, 5.9032e-13,\n 6.7091e-12, 4.5867e-13, 1.6024e-12, 2.1780e-12, 3.8595e-12, 6.4313e-13,\n 4.7825e-12, 1.4399e-13, 6.0206e-12, 4.1887e-13, 4.3329e-13, 1.8962e-12,\n 4.5006e-13, 1.7820e-13, 2.6883e-12, 6.9349e-14, 5.6394e-13, 3.7686e-12,\n 2.2943e-12, 7.1246e-13, 7.3930e-13, 1.4431e-13, 1.6420e-13, 3.1034e-13,\n 2.0786e-12, 2.4518e-12, 4.7352e-12, 1.4617e-12, 1.9801e-12, 4.8029e-13,\n 4.1130e-12, 1.0445e-12, 5.0149e-13, 4.6387e-13, 6.3115e-12, 4.3704e-12,\n 6.2188e-12, 1.3695e-12, 1.6962e-12, 1.1810e-12, 2.8097e-12, 4.9580e-12,\n 3.2105e-13, 5.0319e-12, 4.9416e-12, 4.3481e-12, 1.8205e-12, 7.0153e-13,\n 8.6700e-14, 5.9148e-14, 3.2439e-13, 6.5365e-12, 5.4861e-13, 4.5009e-12,\n 1.6934e-12, 1.1580e-12, 6.7948e-12, 1.8019e-12, 6.3939e-12, 7.5114e-13,\n 7.1848e-13, 2.9295e-12, 9.0883e-12, 5.5800e-12, 4.6121e-13, 5.0727e-12,\n 2.5000e-13, 6.6925e-12, 3.6814e-12, 4.4534e-12, 4.1918e-12, 2.5873e-13,\n 5.1303e-13, 2.7490e-12, 3.3075e-13, 2.3572e-12, 1.2081e-12, 3.1544e-12,\n 2.1746e-13, 4.8647e-13, 2.6221e-13, 7.7158e-12, 6.9514e-13, 4.1352e-12,\n 1.0271e-13, 2.7423e-12, 1.2529e-12, 1.1432e-13, 1.3463e-12, 1.3528e-12,\n 4.0725e-12, 2.1177e-12, 4.7996e-13, 2.0409e-12, 1.1639e-12, 3.3389e-12,\n 2.0853e-13, 5.9964e-12, 8.5082e-13, 4.7055e-12, 3.2627e-12, 6.7196e-13,\n 3.9306e-13, 4.8330e-12, 3.1147e-12, 4.7831e-12, 6.1759e-12, 1.6548e-13,\n 2.5760e-12, 2.0266e-12, 1.3750e-13, 4.3587e-12, 3.1087e-12, 5.6701e-12,\n 1.1418e-13, 1.4852e-13, 9.0466e-13, 1.0390e-12, 2.6702e-12, 9.2837e-13,\n 2.6527e-12, 1.0212e-11, 9.3635e-13, 7.8303e-12, 2.7211e-12, 2.0504e-12,\n 6.5508e-12, 2.5946e-13, 5.5463e-12, 1.3063e-11, 9.1006e-13, 3.8549e-12,\n 5.3162e-13, 4.7410e-13, 2.9817e-12, 1.6449e-12, 2.7902e-13, 2.7219e-12,\n 3.2289e-12, 2.7901e-13, 8.1474e-13, 3.2972e-13, 1.8743e-12, 9.4479e-14,\n 2.7545e-13, 3.4074e-12, 3.1815e-12, 4.2379e-12, 1.1505e-13, 1.4155e-12,\n 2.6018e-12, 8.9878e-12, 1.1665e-13, 9.4929e-13, 3.4557e-12, 3.1887e-12,\n 2.9757e-13, 2.2203e-12, 9.5673e-13, 6.2071e-13, 1.5124e-12, 1.9000e-11,\n 2.1848e-12, 1.8548e-12, 2.1303e-13, 1.3666e-12, 9.8464e-13, 1.7140e-12,\n 1.8242e-12, 3.3986e-13, 3.9076e-13, 4.3116e-12, 9.8634e-14, 1.1280e-12,\n 2.3543e-12, 5.0090e-12, 2.1762e-12, 9.1857e-13, 3.0958e-14, 2.5336e-13,\n 1.1309e-12, 8.9631e-13, 2.1311e-11, 2.9636e-12, 3.3572e-12, 3.6401e-12,\n 9.2086e-12, 5.6454e-12, 2.0643e-12, 3.9065e-12, 3.3528e-13, 2.6956e-13,\n 1.1354e-13, 4.2162e-13, 3.3080e-12, 5.0992e-12, 1.4162e-12, 4.9857e-13,\n 9.6335e-13, 6.1581e-13, 4.5800e-12, 1.2733e-11, 3.0461e-12, 2.5537e-13,\n 1.4444e-11, 1.3088e-13, 3.2138e-13, 4.6800e-12, 8.7802e-12, 5.3840e-13,\n 1.1240e-12, 1.3669e-11, 5.7058e-12, 8.9291e-12, 3.0525e-12, 1.3922e-12,\n 1.4341e-12, 1.9585e-12, 2.6206e-13, 4.6968e-13, 2.2155e-12, 2.8834e-12,\n 1.3386e-12, 2.4881e-13, 4.4161e-12, 7.1352e-12, 8.9887e-13, 1.9251e-13,\n 1.9573e-12, 2.0995e-13, 8.6703e-13, 5.2517e-14, 4.2999e-12, 2.1440e-13,\n 1.0597e-11, 4.8012e-12, 1.2535e-12, 7.5621e-12, 1.5405e-12, 1.9455e-12,\n 8.3735e-14, 2.1548e-12, 1.3303e-12, 7.4734e-13, 2.9824e-12, 8.7239e-12,\n 4.5951e-14, 7.1133e-12, 3.9937e-13, 2.1213e-12, 7.0679e-13, 1.0247e-12,\n 1.9712e-12, 3.4304e-12, 1.2008e-12, 2.0279e-12, 3.7129e-13, 2.0275e-13,\n 2.2463e-12, 2.7766e-12, 3.8672e-13, 4.8629e-13, 3.6390e-13, 2.9556e-12,\n 1.0073e-12, 4.9844e-12, 1.8028e-12, 1.7717e-13, 1.2951e-13, 5.0497e-12,\n 4.0722e-12, 4.0835e-12, 6.7555e-13, 2.5321e-12, 3.9555e-12, 4.0351e-12,\n 5.0053e-13, 1.7439e-12, 8.6678e-13, 3.1017e-12, 7.7694e-13, 2.6816e-12,\n 4.5327e-13, 9.8358e-12, 3.6521e-12, 3.0618e-12, 1.1380e-13, 1.2695e-12,\n 1.8287e-13, 3.9603e-12, 5.7406e-13, 4.8335e-12, 6.0299e-13, 1.9630e-12,\n 1.9377e-12, 1.1513e-11, 3.2358e-13, 5.5141e-12, 3.2297e-13, 1.6325e-12,\n 6.1646e-13, 4.4556e-12, 4.9412e-12, 8.4695e-14, 2.3449e-12, 9.7997e-14,\n 1.0853e-12, 2.2339e-12, 5.9734e-13, 3.0954e-12, 7.0747e-14, 1.4428e-11,\n 6.8218e-14, 1.8380e-12, 2.9746e-12, 2.8999e-12, 3.2952e-12, 1.7713e-12,\n 1.0229e-12, 1.3685e-13, 1.4549e-13, 1.0441e-12, 1.1920e-13, 9.7340e-13,\n 1.9456e-13, 6.6271e-12, 1.1622e-12, 1.6877e-11, 7.9220e-12, 3.4456e-12,\n 3.1398e-12, 1.0789e-11, 1.9910e-12, 2.4094e-12, 2.1900e-12, 1.6492e-12,\n 2.6922e-12, 6.4029e-12, 1.1805e-13, 2.2689e-12, 6.0273e-12, 3.1898e-12,\n 5.8976e-12, 1.3424e-12, 1.6656e-13, 7.5212e-14, 4.4911e-12, 1.8897e-12,\n 4.5104e-14, 1.6691e-13, 4.7685e-12, 2.9936e-14, 4.6533e-12, 4.1897e-12,\n 1.6919e-13, 1.2876e-13, 1.5928e-12, 1.0514e-11, 2.5061e-13, 8.4221e-12,\n 1.5295e-13, 4.4502e-12, 4.3474e-12, 7.2324e-13, 2.1521e-12, 9.1188e-14,\n 6.1150e-13, 4.3856e-13, 2.5307e-12, 5.5593e-13, 2.6929e-13, 1.2896e-12,\n 4.7487e-12, 1.7775e-12, 2.0597e-12, 2.7855e-12, 6.2992e-12, 1.4809e-12,\n 2.5753e-12, 7.9026e-12, 7.6231e-13, 8.5142e-13, 4.2954e-13, 4.7566e-12,\n 1.4992e-12, 1.6772e-12, 1.4593e-12, 1.7340e-12, 3.8816e-12, 2.3582e-12,\n 2.1546e-12, 6.5966e-12, 2.4564e-12, 3.4651e-13, 2.2810e-12, 2.7295e-12,\n 4.9587e-12, 2.8020e-13, 4.2299e-12, 2.2416e-12, 4.2668e-13, 7.1571e-13,\n 3.8122e-13, 1.9834e-12, 2.7775e-12, 4.8428e-12, 4.6241e-14, 7.1510e-13,\n 1.0907e-13, 1.1551e-12, 1.7221e-12, 7.3189e-13, 7.5574e-12, 5.9218e-14,\n 7.3762e-13, 3.9884e-12, 2.5572e-12, 5.8045e-12, 3.0142e-13, 2.5976e-13,\n 1.3619e-12, 6.6833e-13, 3.2349e-12, 5.5893e-14, 1.9042e-12, 1.0428e-11,\n 3.7818e-12, 2.0861e-13, 2.8117e-14, 1.5873e-12, 1.1427e-13, 1.7050e-11,\n 1.4125e-12, 1.5293e-13, 4.9000e-14, 3.9339e-12, 1.4536e-12, 3.4524e-13,\n 1.4712e-12, 1.1437e-12, 4.5004e-13, 2.5635e-12, 1.1867e-13, 2.0431e-13,\n 2.4826e-13, 2.0142e-13, 5.2825e-13, 4.6589e-13, 5.8918e-12, 2.5072e-12,\n 4.5264e-13, 1.6632e-12, 3.7568e-12, 1.6681e-13, 8.2093e-14, 2.2120e-13,\n 5.5809e-12, 5.0664e-12, 5.0297e-13, 1.3371e-13, 1.1465e-13, 4.9427e-12,\n 5.4601e-13, 2.1127e-12, 1.2238e-13, 1.4890e-13, 2.0090e-13, 1.5010e-13,\n 2.8797e-12, 6.9655e-13, 1.3122e-11, 6.3000e-12, 2.6701e-12, 7.7180e-13,\n 3.7735e-14, 2.5945e-12, 2.7675e-13, 2.2418e-12, 2.5977e-13, 3.5812e-13,\n 1.3945e-12, 1.1247e-12, 5.3873e-12, 7.1484e-13, 1.4273e-12, 2.1029e-13,\n 1.3534e-11, 3.0179e-14, 1.5924e-12, 3.0318e-13, 1.3812e-13, 1.1886e-12,\n 3.8957e-13, 2.3185e-13, 2.9660e-12, 3.0148e-12, 1.7842e-12, 1.9791e-12,\n 1.1159e-13, 2.8769e-12, 8.9539e-14, 2.3171e-12, 2.8414e-11, 1.1822e-13,\n 5.4848e-13, 2.0247e-12, 3.5509e-12, 5.0734e-13, 7.8822e-13, 1.3258e-12,\n 7.2364e-12, 3.0699e-12, 6.6446e-13, 4.3100e-12, 6.7249e-14, 1.3856e-12,\n 6.5458e-12, 1.0976e-12, 4.4446e-12, 8.1662e-14, 1.1918e-12, 2.6164e-13,\n 3.1491e-13, 2.9472e-12, 3.4424e-12, 1.9435e-12, 1.6086e-11, 1.0368e-12,\n 6.6187e-12, 8.7625e-13, 1.2325e-12, 8.4119e-13, 1.1419e-12, 3.8384e-12,\n 1.5612e-13, 5.7024e-13, 1.9714e-12, 5.2884e-12], device='cuda:0')" + }, + "56": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 8.3830e-15, 4.6675e-14, 3.0571e-13, 1.4189e-13, 4.8891e-14,\n -2.7086e-16, -8.1980e-16, -1.1483e-14, -6.9726e-15, -6.1282e-15,\n 3.9492e-13, -1.0101e-14, -1.4749e-14, 6.1712e-15, 3.3089e-13,\n 1.2511e-13, -1.6814e-15, -6.5219e-15, 6.6607e-14, -1.0895e-15,\n -8.9428e-15, 1.5643e-14, -4.7944e-15, 3.6180e-14, 1.7274e-14,\n 7.2822e-14, 4.5757e-14, -9.0575e-14, 5.0651e-14, -2.7494e-15,\n -3.3673e-15, 1.0017e-14, 1.0682e-14, 2.7360e-13, -7.1332e-15,\n -5.1018e-15, -1.3071e-14, -1.8793e-14, -3.1706e-15, -8.6374e-15,\n -1.3312e-14, 1.2900e-14, -2.3725e-15, 1.9044e-15, -1.1901e-15,\n 1.8798e-13, 1.3487e-13, 1.1311e-13, -1.3545e-14, -2.7013e-16,\n -9.5381e-15, -8.6070e-16, 1.0212e-14, 2.5321e-15, -1.6310e-14,\n -8.3227e-15, 6.4651e-14, 5.8334e-14, 4.7976e-15, 3.8541e-14,\n -1.7600e-15, 1.3023e-14, 2.0556e-15, 5.5040e-15, 1.3074e-13,\n 6.2749e-14, 9.6671e-16, 1.6811e-14, 3.6015e-15, 1.0874e-13,\n 2.2965e-13, 4.4135e-14, 1.8295e-14, 7.0177e-15, -1.2233e-14,\n 9.8018e-16, -2.7980e-16, 1.6158e-14, 1.7125e-15, 3.5542e-14,\n -4.0774e-16, 9.0564e-15, 2.6728e-14, 4.1732e-13, -4.5030e-15,\n 6.8196e-14, -6.0022e-15, -1.2449e-14, -7.8942e-16, -6.0191e-15,\n 2.1032e-14, 1.1162e-13, -5.9724e-14, -9.0170e-15, -1.2945e-14,\n -5.8987e-15, -3.3791e-15, -9.6972e-17, 9.3133e-14, 1.4529e-13,\n -1.1425e-16, -1.4428e-15, 3.5793e-15, 7.0643e-14, -1.7135e-14,\n -9.2194e-15, 8.9969e-16, 2.7552e-14, 3.0613e-13, -1.2326e-16,\n 2.2442e-14, 1.7688e-15, 1.2156e-14, 1.6350e-13, 1.3679e-14,\n -2.6884e-15, 4.9314e-14, -1.1306e-14, 3.0348e-15, -1.0518e-14,\n 1.3541e-14, 9.7539e-15, 2.3630e-13, 1.4195e-13, 1.1660e-13,\n -3.8597e-17, 2.3377e-14, -1.5556e-13, 3.7703e-13, 1.9598e-13,\n -8.0231e-15, 1.0708e-15, 9.9585e-15, 6.7434e-14, 1.0006e-14,\n 1.4538e-13, 9.2172e-14, -5.7568e-14, 5.2170e-13, -1.2620e-14,\n -3.8132e-15, 6.9798e-15, -7.2162e-15, 1.4277e-13, 1.3981e-14,\n 1.5455e-13, 7.6776e-16, 1.2933e-13, -7.8653e-15, -3.0703e-16,\n 2.8349e-14, 5.0017e-14, 4.9127e-15, -1.6214e-14, 2.1112e-15,\n 1.2748e-13, 7.3701e-15, 9.5871e-14, 6.0220e-14, 9.5497e-14,\n 5.0755e-14, 7.5472e-14, -1.2636e-14, -1.5617e-14, -1.3114e-15,\n 9.1666e-14, -5.7651e-15, 3.5656e-15, 1.3970e-14, 1.5270e-14,\n -1.7209e-14, 3.1659e-15, 8.1171e-15, 4.5375e-14, -1.4799e-14,\n 6.8737e-15, 7.1556e-14, -2.1865e-15, 1.5835e-15, -6.6641e-16,\n 2.1771e-15, 2.4340e-13, -1.7239e-14, 1.2535e-13, -1.7208e-15,\n -5.3915e-15, -3.1224e-15, -9.9145e-14, 3.0462e-17, -4.4589e-15,\n 2.4114e-14, 2.2088e-14, 2.2300e-14, 4.0813e-16, -1.5171e-14,\n 2.5588e-14, -9.0145e-15, 4.6511e-14, -2.3263e-15, -7.5230e-16,\n 4.1621e-14, -2.4390e-14, 3.4621e-14, 6.2858e-16, 2.3788e-13,\n -3.4049e-15, -2.6046e-15, -2.5034e-14, 1.9469e-13, 4.8791e-15,\n -2.3364e-15, 1.6195e-13, 4.1861e-15, 3.6607e-14, 1.0096e-13,\n -1.2380e-14, 2.2959e-15, -6.8837e-15, 1.4316e-13, -1.8999e-14,\n 1.1404e-13, 1.0585e-13, 4.1164e-15, 6.4758e-15, 6.5093e-16,\n 1.6869e-13, -3.5448e-15, 3.6109e-15, 3.4598e-13, -5.1810e-15,\n -5.9079e-15, 5.6480e-14, 1.3117e-15, 1.1613e-13, 1.2404e-14,\n 5.8459e-16, 8.2089e-15, -1.9702e-14, 2.8587e-14, 1.7206e-14,\n 2.4817e-13, -6.4211e-15, 3.4786e-14, 2.8439e-15, -1.0202e-14,\n 2.0411e-14, 3.1328e-13, 5.5640e-14, -1.4877e-14, 1.3604e-13,\n -1.4331e-15, -1.6262e-13, -2.7360e-14, 2.4224e-14, 2.8664e-14,\n 6.2116e-14, 1.3316e-15, 4.4868e-14, -7.3403e-15, -2.2700e-15,\n -2.3980e-15, 2.9087e-14, 1.9320e-13, 2.1524e-13, -5.5494e-15,\n -1.4285e-15, -5.7894e-15, 3.7248e-13, -1.2101e-14, 3.2102e-13,\n 2.9810e-13, 9.5202e-14, -4.1115e-15, 9.5269e-14, 1.8325e-14,\n -3.6890e-15, -7.7917e-14, 4.4138e-14, -3.5192e-14, 7.3918e-14,\n 4.3585e-15, 9.0497e-15, -9.9435e-15, 2.8444e-14, -1.0028e-14,\n 5.9363e-14, 7.9428e-15, 7.5845e-16, -1.8000e-13, 1.6932e-14,\n 1.2891e-14, -4.8617e-15, -2.5540e-15, 1.3171e-13, -8.3591e-15,\n -1.1567e-14, -9.9565e-15, 8.7638e-15, -6.2820e-14, 3.0852e-14,\n -3.9460e-15, -4.6717e-15, -2.2449e-14, -1.9738e-15, 8.6875e-15,\n 1.5464e-14, 1.5155e-14, -2.3750e-14, 1.4488e-14, 2.0409e-13,\n -7.7116e-15, 9.1529e-15, 1.0676e-13, 2.8034e-13, 4.6542e-14,\n 5.9358e-14, 2.0168e-15, 1.0434e-13, -7.3943e-14, -1.0180e-14,\n 1.8462e-14, 8.1314e-14, 5.4602e-15, 9.1831e-15, 2.3061e-13,\n 1.2500e-13, 3.4264e-13, 3.1023e-13, 1.2040e-13, 5.4484e-14,\n 3.2539e-13, 2.4975e-13, -5.7317e-15, -1.0082e-15, 2.0514e-13,\n 5.3914e-16, 2.7388e-14, -2.4131e-15, 6.8623e-14, 8.7989e-14,\n 1.3237e-13, 4.4945e-14, 2.9972e-14, 3.2286e-14, -1.4122e-15,\n 1.8975e-15, 9.4602e-14, 1.1294e-13, -1.3063e-15, -7.9564e-15,\n 4.4949e-14, 5.5589e-14, -1.6949e-14, -9.7693e-15, 2.3611e-14,\n 4.3779e-14, 1.1012e-13, -6.9461e-16, 1.9954e-15, 1.8856e-14,\n 2.0826e-13, 1.7116e-13, 1.1546e-15, -8.3044e-14, 1.3746e-13,\n 1.3278e-13, -6.5859e-15, 3.2936e-14, -2.7707e-15, -1.0440e-15,\n -1.1189e-15, 5.3256e-13, 1.7071e-13, 2.5289e-14, 7.3201e-14,\n 2.3460e-14, 6.0433e-15, 6.7981e-14, -5.4785e-15, 1.7956e-16,\n -8.5006e-15, -2.3825e-14, 1.0399e-13, 5.7777e-17, 3.5508e-14,\n 2.4437e-14, 4.0424e-13, 1.5692e-13, 5.7674e-15, -1.0023e-14,\n 2.6712e-14, -1.3821e-14, -2.2560e-14, 7.0681e-14, 1.6646e-13,\n -5.8252e-14, 1.0139e-13, 1.3611e-14, 4.0175e-16, 4.8227e-14,\n -1.2051e-14, 3.4332e-16, -3.1092e-16, -4.7884e-15, 7.7590e-14,\n -5.8073e-15, 9.9575e-15, 1.2392e-15, 3.3689e-15, -1.4510e-14,\n 3.0318e-13, 1.5522e-13, 1.2841e-14, 1.3970e-14, 3.5895e-15,\n 8.5394e-14, 6.3473e-13, -1.9263e-15, 2.2595e-14, -3.2432e-15,\n -8.0780e-15, 3.5946e-14, -3.9004e-15, -6.9821e-15, 3.2952e-16,\n 2.4303e-13, -1.2331e-14, 1.4246e-13, -5.3054e-15, -1.7039e-13,\n 4.3931e-14, 3.1331e-13, 2.2731e-13, 5.4728e-14, 5.8323e-14,\n 6.2302e-14, 1.4473e-15, 5.2647e-15, 1.8904e-14, -6.7100e-14,\n 1.0374e-14, -1.9651e-16, 9.0518e-15, 6.3309e-15, 1.1346e-13,\n -4.4727e-15, 9.0333e-15, -5.7405e-16, 9.3366e-15, -4.2689e-15,\n 2.7861e-15, 1.8321e-14, 3.2286e-13, 2.2548e-14, 8.7956e-15,\n -1.2978e-15, 3.2188e-14, 2.8872e-14, -2.0561e-14, 4.6082e-14,\n 2.9082e-14, 1.5114e-14, -9.6007e-16, 4.6840e-14, 8.3695e-15,\n -4.4808e-16, -6.3339e-16, 7.5705e-15, 2.6116e-15, -4.5674e-16,\n -2.2712e-14, -1.0673e-14, -7.7451e-15, -2.1712e-15, 1.5569e-13,\n 5.2489e-14, -1.5060e-13, 7.7745e-14, 1.5330e-13, -6.7479e-15,\n 7.5238e-14, -2.2610e-14, 3.6339e-14, 1.2157e-15, 5.3524e-13,\n 1.1352e-14, 6.1271e-14, 1.5640e-14, 2.1441e-13, -9.0409e-15,\n 1.6789e-14, -1.6125e-14, -4.2137e-15, 9.6769e-14, -3.4123e-15,\n -1.2419e-14, -1.2321e-14, 6.0674e-15, -1.0825e-14, -5.3370e-16,\n 4.8781e-15, -9.3057e-15, 3.4336e-14, 4.4677e-14, 3.0951e-14,\n 1.9134e-13, 1.2644e-14, 3.7610e-14, 9.4333e-16, 4.9423e-14,\n 4.6765e-15, 2.5576e-13, -1.0504e-15, 8.4543e-15, 1.0099e-14,\n 8.7889e-15, 9.4697e-15, -7.6240e-14, 2.5368e-13, 4.5107e-14,\n 3.5795e-15, 3.0245e-15, 1.0757e-15, 7.8194e-14, 6.5106e-15,\n 6.2838e-15, 3.4582e-16, 1.0508e-13, -3.9462e-15, 2.7578e-15,\n 1.9931e-15, -2.8476e-14, 1.6177e-15, 2.2901e-13, 5.0229e-14,\n -4.6310e-15, 4.5465e-14, 4.6071e-14, -3.1836e-15, 2.6346e-14,\n -1.4082e-14, 4.7429e-14, 8.4581e-14, 5.5082e-14, 8.8017e-16,\n 2.8162e-15, 9.5752e-15, -5.7731e-15, 1.5852e-14, -4.9884e-15,\n 3.6233e-13, -1.6909e-14, -1.0536e-15, 6.9434e-14, -6.7899e-14,\n 1.4207e-14, 1.1502e-14, 5.9448e-15, 1.9774e-13, 3.5792e-14,\n 2.3396e-13, 1.0538e-16, 1.6450e-14, 1.9453e-13, 2.7572e-15,\n 2.3290e-14, -6.7361e-15, -4.8860e-15, -3.5797e-15, 1.1558e-14,\n 4.1187e-14, 2.4017e-14, -1.6529e-14, 3.2578e-13, -2.4858e-14,\n -2.9947e-15, -1.1079e-15, 2.4509e-14, 2.4451e-14, -2.3355e-14,\n 3.7036e-15, 4.1052e-15, -1.9425e-14, 7.0974e-15, 5.2663e-13,\n 1.4342e-15, -1.1095e-15, 8.9580e-15, -1.4528e-14, -6.4547e-15,\n 6.5060e-14, 8.4554e-15, 1.0059e-14, 2.5143e-14, -1.1009e-16,\n 3.9845e-13, 2.0355e-13, -2.2648e-14, 3.6530e-14, 2.2196e-14,\n 5.4982e-15, 7.7781e-15, 9.4218e-14, 2.3412e-13, 5.9016e-14,\n 7.2155e-15, -7.8013e-15, -2.1910e-15, 9.0091e-15, 4.1973e-16,\n 1.1111e-14, 5.3307e-15, 2.6615e-14, 8.5753e-14, -1.6223e-15,\n 1.7073e-13, -3.2547e-15, -1.0056e-14, 6.9628e-14, 8.0419e-13,\n 2.8435e-14, 3.1078e-13, 4.4995e-14, 7.2645e-14, 2.3380e-15,\n -1.8749e-14, 4.4343e-14, 6.8531e-15, 1.1483e-13, 4.1249e-15,\n 1.9650e-14, -2.5982e-15, 8.1610e-15, 1.7063e-13, 1.3226e-14,\n -1.9404e-14, 2.1471e-14, 3.4482e-14, -1.1815e-14, 2.8909e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([2.9927e-17, 5.3638e-18, 3.4252e-15, 1.9977e-16, 1.1097e-16, 1.3445e-14,\n 9.9032e-17, 1.8028e-16, 4.0633e-16, 5.0509e-17, 1.5207e-16, 1.8966e-14,\n 1.5512e-15, 1.4402e-16, 1.8926e-16, 8.3880e-17, 7.4185e-16, 2.2012e-16,\n 1.1890e-17, 1.4099e-17, 2.5895e-17, 3.6735e-17, 1.8472e-14, 3.9675e-15,\n 2.8865e-15, 9.1119e-15, 4.1127e-15, 2.5494e-15, 2.3792e-15, 2.1702e-16,\n 9.7221e-16, 2.0216e-16, 2.7190e-15, 1.7669e-15, 3.8677e-17, 1.2794e-16,\n 4.1183e-17, 3.3432e-17, 8.7106e-15, 2.4813e-16, 8.8059e-18, 2.0638e-16,\n 6.8003e-17, 6.7307e-18, 5.0912e-17, 1.3920e-15, 9.2510e-15, 4.3725e-16,\n 8.8092e-18, 8.1788e-17, 1.1247e-16, 8.7186e-17, 4.3866e-17, 5.8274e-16,\n 2.5376e-16, 1.2007e-17, 2.6195e-16, 1.1053e-15, 5.3247e-17, 2.2110e-17,\n 9.9510e-17, 1.3561e-16, 2.4662e-17, 7.8136e-16, 1.2706e-14, 4.6269e-15,\n 1.3613e-16, 1.2068e-15, 1.7355e-18, 5.6900e-15, 3.4399e-15, 2.4353e-15,\n 2.3559e-15, 5.5574e-15, 1.5347e-15, 1.6522e-16, 3.3806e-17, 8.4698e-17,\n 8.0003e-17, 2.5846e-16, 2.1709e-15, 3.6484e-16, 1.3208e-17, 4.0467e-15,\n 3.7903e-17, 5.8783e-16, 9.3959e-16, 6.5080e-17, 1.6672e-15, 7.3036e-17,\n 3.1161e-16, 9.2882e-16, 3.0677e-14, 4.4861e-17, 1.8639e-16, 1.6265e-16,\n 2.3105e-16, 1.5101e-17, 9.5441e-16, 1.8785e-15, 4.2950e-16, 2.2265e-17,\n 3.9761e-15, 1.3869e-16, 6.0564e-16, 6.1420e-16, 4.2945e-15, 3.6090e-17,\n 3.5289e-15, 1.5117e-17, 1.0336e-14, 1.9185e-17, 1.2168e-16, 2.2440e-16,\n 1.2475e-16, 5.5550e-17, 1.2959e-15, 8.8392e-17, 3.3102e-17, 5.5412e-16,\n 2.4949e-16, 8.3503e-17, 6.3366e-17, 1.0741e-17, 1.9530e-17, 2.6446e-17,\n 2.0266e-16, 2.2885e-15, 1.4004e-14, 2.4418e-16, 6.2488e-16, 1.5295e-17,\n 4.3563e-16, 2.7043e-17, 7.0771e-17, 7.0521e-17, 4.2571e-15, 6.5816e-15,\n 5.3219e-15, 2.8842e-16, 8.2814e-16, 3.0886e-16, 6.2326e-16, 5.6123e-15,\n 2.5629e-16, 1.5686e-15, 3.2042e-15, 1.0135e-15, 4.2041e-16, 3.9457e-17,\n 6.7344e-17, 5.6559e-18, 4.6226e-17, 1.3709e-15, 3.6921e-17, 4.9888e-15,\n 6.5827e-16, 5.8696e-17, 1.0518e-14, 5.6447e-16, 5.6040e-15, 1.4036e-16,\n 1.0599e-16, 7.4770e-16, 4.3107e-15, 3.4382e-15, 1.3319e-16, 2.5728e-15,\n 6.2917e-17, 2.6919e-15, 1.1898e-15, 2.7282e-15, 1.0483e-15, 6.3864e-17,\n 4.7250e-17, 2.8943e-16, 2.6120e-17, 5.9852e-16, 4.6007e-17, 1.1849e-15,\n 2.3529e-17, 2.6573e-17, 2.1274e-17, 6.7328e-15, 1.3918e-17, 1.8007e-16,\n 1.2208e-17, 8.5581e-16, 4.8567e-16, 6.6589e-18, 2.8148e-16, 2.9908e-17,\n 1.3657e-15, 1.1698e-15, 2.9719e-17, 2.8782e-16, 8.3306e-17, 6.8907e-16,\n 2.8915e-17, 1.8996e-15, 2.7044e-16, 4.0135e-15, 2.3153e-15, 1.5456e-16,\n 8.8049e-17, 4.3954e-15, 2.8118e-15, 1.3998e-14, 9.1662e-15, 4.8216e-17,\n 4.8167e-16, 5.1498e-16, 1.8889e-17, 3.0308e-15, 4.2997e-15, 4.4473e-15,\n 2.9040e-17, 5.8956e-17, 1.2793e-16, 6.4643e-17, 5.1298e-15, 7.5431e-16,\n 6.8312e-16, 1.5233e-14, 2.7856e-16, 9.3015e-15, 1.4152e-15, 1.7062e-16,\n 6.7759e-15, 2.6996e-16, 2.0981e-15, 6.5767e-15, 3.1211e-16, 8.3683e-16,\n 6.6195e-17, 2.4249e-17, 8.1389e-16, 1.8116e-15, 5.9578e-17, 2.8840e-15,\n 1.9926e-15, 1.9900e-17, 4.1368e-17, 9.3258e-18, 5.5376e-16, 5.3080e-18,\n 6.7176e-18, 7.4387e-16, 6.1398e-16, 1.2823e-15, 2.4068e-17, 3.5430e-16,\n 8.4822e-16, 7.5563e-15, 5.2569e-17, 5.3226e-16, 1.0168e-15, 7.3357e-16,\n 4.0589e-17, 8.1652e-16, 1.0359e-16, 8.5141e-18, 1.1012e-15, 2.1777e-14,\n 3.6259e-16, 1.0054e-15, 2.4640e-17, 2.0096e-16, 1.0367e-16, 1.3985e-15,\n 1.9331e-15, 2.5732e-17, 1.1805e-17, 2.2131e-15, 1.1675e-16, 2.2275e-17,\n 5.7530e-15, 1.8261e-15, 1.2794e-15, 5.7955e-17, 5.1516e-18, 1.7607e-17,\n 2.3797e-16, 1.6405e-17, 8.2507e-15, 1.3595e-15, 4.1063e-15, 7.3490e-16,\n 8.0687e-15, 1.9029e-15, 8.8331e-17, 2.4577e-15, 1.0836e-17, 1.4877e-17,\n 6.1573e-17, 1.7559e-17, 4.4765e-16, 2.2281e-15, 2.9532e-16, 1.6360e-17,\n 1.3270e-16, 1.9827e-17, 1.1516e-15, 8.5990e-15, 6.7665e-15, 2.7756e-17,\n 3.5718e-15, 4.3092e-16, 9.9529e-18, 1.6543e-15, 1.3787e-14, 6.2304e-17,\n 3.9809e-15, 6.3067e-15, 2.4630e-15, 3.7863e-15, 3.4624e-16, 3.7667e-16,\n 4.8296e-16, 2.8441e-16, 2.3209e-17, 1.3780e-16, 1.1329e-15, 1.8284e-15,\n 7.6818e-16, 5.9720e-17, 2.3604e-15, 5.1835e-15, 2.2420e-17, 2.0724e-16,\n 5.9042e-16, 1.1241e-17, 1.1878e-16, 2.2309e-18, 7.9897e-16, 6.3009e-18,\n 6.5616e-15, 1.6315e-15, 2.0474e-16, 4.3340e-15, 1.1888e-15, 1.0035e-15,\n 1.2299e-16, 2.4392e-16, 4.9150e-16, 4.1665e-17, 1.7519e-15, 8.5542e-15,\n 4.6009e-18, 2.6914e-15, 3.7145e-17, 7.9662e-16, 1.9200e-16, 9.8542e-17,\n 9.1138e-17, 1.3453e-15, 5.6328e-16, 8.5641e-16, 1.5699e-17, 2.1194e-17,\n 1.1303e-15, 1.4440e-15, 5.3171e-17, 6.3822e-17, 8.8595e-17, 5.6939e-15,\n 7.0690e-17, 4.7648e-15, 4.3759e-17, 1.2329e-17, 7.5349e-17, 7.6477e-15,\n 2.4628e-15, 2.5819e-15, 3.7962e-17, 1.3689e-16, 7.8763e-16, 1.7601e-15,\n 3.4696e-17, 4.1120e-17, 1.3494e-16, 3.3865e-15, 8.4819e-17, 6.7571e-16,\n 4.1591e-18, 4.1821e-15, 1.6776e-14, 1.5531e-15, 1.1209e-16, 5.7634e-16,\n 4.5515e-17, 1.5832e-16, 8.6302e-17, 2.5715e-15, 1.4974e-17, 6.9497e-16,\n 3.4126e-16, 1.0851e-14, 6.8453e-18, 3.1626e-15, 5.4142e-17, 2.5586e-16,\n 7.1154e-17, 7.6314e-16, 7.8452e-15, 3.3983e-18, 7.6581e-15, 8.1467e-17,\n 4.8560e-17, 1.9682e-16, 1.8254e-16, 3.0639e-15, 2.2622e-17, 9.0270e-15,\n 8.1266e-18, 4.8505e-15, 7.3805e-15, 2.7300e-16, 2.4309e-16, 1.5321e-15,\n 2.5784e-16, 1.1214e-16, 9.5732e-17, 4.6482e-16, 1.3176e-17, 8.6709e-17,\n 1.3829e-16, 3.1444e-15, 4.2208e-17, 4.5860e-14, 1.1468e-14, 1.8808e-15,\n 1.9164e-15, 3.2780e-15, 6.6369e-15, 2.9485e-16, 3.7704e-16, 2.9055e-16,\n 3.6372e-15, 2.6938e-15, 7.1600e-18, 1.5301e-15, 3.3851e-15, 7.4561e-15,\n 1.5035e-15, 7.3192e-16, 1.9001e-16, 1.6579e-17, 5.4709e-16, 8.3608e-17,\n 7.9842e-18, 3.2424e-18, 5.9256e-15, 1.0672e-17, 3.7339e-15, 2.8538e-15,\n 5.4915e-17, 6.5796e-17, 1.9109e-16, 1.8985e-14, 1.7618e-16, 4.8280e-15,\n 1.0078e-16, 5.2644e-15, 2.7193e-15, 5.0468e-18, 2.6064e-15, 1.0854e-16,\n 6.4366e-17, 3.4001e-18, 7.2059e-16, 1.5835e-16, 9.9397e-17, 1.4802e-16,\n 2.3808e-15, 2.1431e-15, 1.1350e-15, 2.8004e-16, 1.7060e-14, 1.9648e-16,\n 3.6525e-15, 5.4347e-15, 2.2254e-17, 5.7333e-18, 6.2528e-17, 1.9254e-15,\n 1.2028e-16, 5.1938e-16, 1.2640e-15, 1.1971e-16, 2.7490e-16, 2.0455e-16,\n 1.5401e-15, 2.4627e-14, 1.7785e-15, 1.2904e-16, 3.3154e-16, 3.6354e-15,\n 2.1960e-15, 2.1314e-17, 2.8144e-15, 1.1042e-15, 2.1172e-17, 9.4588e-18,\n 3.5157e-18, 1.9833e-16, 3.1750e-16, 3.1476e-15, 2.1930e-17, 4.6207e-16,\n 1.3544e-16, 2.9744e-16, 2.4459e-16, 2.5627e-17, 1.5880e-15, 3.2159e-16,\n 6.9892e-18, 8.9199e-16, 6.8790e-16, 9.8700e-15, 2.7331e-17, 5.3248e-18,\n 1.9892e-16, 9.4329e-17, 1.9017e-15, 1.2448e-17, 2.2073e-15, 1.4128e-14,\n 5.2123e-15, 1.4698e-17, 8.2486e-18, 5.8311e-16, 1.1463e-16, 1.5369e-14,\n 1.1281e-16, 3.9347e-18, 4.4605e-17, 1.7666e-15, 1.5312e-16, 1.0393e-17,\n 1.1533e-16, 5.3695e-17, 7.4254e-18, 1.5479e-15, 5.9986e-17, 1.5310e-17,\n 2.5600e-17, 1.1201e-16, 2.2220e-16, 1.3101e-17, 4.4480e-15, 2.1211e-15,\n 5.1991e-17, 3.9213e-16, 1.5300e-14, 2.5551e-17, 1.5048e-16, 8.2682e-17,\n 5.5584e-16, 2.6907e-15, 1.0335e-16, 5.6066e-17, 1.0383e-16, 5.0579e-15,\n 1.1647e-16, 2.4788e-16, 1.8124e-18, 1.0683e-17, 1.1944e-16, 1.2394e-16,\n 1.5212e-15, 8.4615e-17, 8.9180e-15, 6.5580e-15, 2.9681e-16, 1.8942e-17,\n 7.7425e-18, 1.8187e-15, 7.1074e-17, 6.6226e-16, 5.4718e-17, 1.8902e-17,\n 7.3468e-17, 3.4369e-16, 9.6714e-15, 2.8920e-17, 1.0499e-15, 1.6406e-16,\n 2.3656e-14, 3.8305e-17, 8.0512e-17, 5.7748e-17, 7.6905e-17, 2.0302e-16,\n 1.5346e-17, 1.2876e-17, 1.6177e-15, 3.0094e-15, 1.6218e-16, 3.7988e-16,\n 3.8698e-17, 1.0664e-15, 3.4257e-17, 7.1138e-16, 2.6002e-14, 5.0628e-18,\n 1.8412e-17, 3.9668e-16, 1.4678e-15, 3.2970e-17, 1.0259e-16, 3.1889e-17,\n 1.0550e-15, 1.2669e-15, 2.3646e-16, 7.4242e-16, 1.4276e-16, 8.5152e-16,\n 4.2105e-15, 8.2059e-16, 2.9078e-15, 3.7396e-18, 1.9080e-16, 8.3357e-17,\n 1.0770e-17, 4.2506e-15, 1.1640e-15, 1.7141e-16, 1.0607e-14, 2.8384e-17,\n 1.1137e-14, 1.5939e-16, 2.1217e-15, 3.3017e-16, 3.2765e-16, 5.9030e-16,\n 2.0637e-17, 1.0390e-16, 8.7282e-16, 3.3529e-15], device='cuda:0')" + }, + "57": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-7.9632e-15, 1.0625e-13, 1.2839e-13, 2.3425e-13, 1.2665e-13,\n 1.4734e-14, -3.2009e-15, 7.6511e-14, 9.3894e-14, 1.2016e-14,\n 3.8745e-13, -2.5241e-14, 4.4072e-14, 1.6303e-14, 3.6559e-13,\n 1.5692e-13, 8.2963e-14, 1.4254e-14, 1.7655e-13, 2.5334e-16,\n 2.7081e-14, -6.2991e-15, -6.4960e-15, 5.0809e-14, 1.2496e-13,\n 1.4852e-13, 1.5771e-13, -7.1672e-14, 1.7567e-13, 5.1792e-15,\n 9.3829e-14, 1.2881e-13, 3.9252e-14, 2.5911e-13, 7.5179e-14,\n -1.1876e-14, 5.4141e-14, 6.0956e-14, 1.2634e-13, 8.1667e-14,\n 4.8880e-14, -1.1829e-14, 1.0586e-14, -8.1719e-15, 2.6976e-15,\n 2.5805e-13, 1.3246e-13, 1.7687e-13, 5.3129e-14, 4.6667e-14,\n 2.7206e-14, 7.6010e-15, -1.8490e-14, -6.4719e-15, 4.6412e-14,\n 1.3548e-14, 1.2997e-13, 1.6042e-13, 4.9269e-14, -2.9149e-14,\n 9.1085e-16, -1.2785e-14, -6.0355e-16, 8.5263e-14, 1.4316e-13,\n 8.3495e-14, -7.8623e-16, 1.7259e-13, -6.1616e-15, 2.1351e-13,\n 3.2307e-13, 1.9753e-13, 5.8866e-14, 4.4263e-14, 5.7746e-14,\n 1.3595e-15, 4.7478e-14, -8.9861e-15, -5.1572e-16, 5.2356e-14,\n 4.9299e-14, 6.8498e-14, -3.0313e-14, 2.2499e-13, 8.3462e-17,\n 1.2098e-13, 4.2646e-14, 3.7015e-14, 3.8021e-15, 1.2736e-14,\n 9.1758e-14, 1.5418e-13, -2.9840e-14, -4.0455e-14, 3.0393e-14,\n 1.8876e-14, 1.3037e-14, 1.7025e-15, 3.5571e-14, 1.5014e-13,\n 8.8352e-17, 2.3443e-15, 7.0291e-14, 1.9130e-13, 4.8124e-14,\n -2.3428e-14, 7.7135e-14, 1.2544e-13, 3.0562e-13, 1.8865e-14,\n 1.7108e-14, -4.7060e-15, -1.0290e-14, 1.2441e-13, 1.2388e-13,\n 1.1023e-15, 8.9873e-14, 1.7523e-14, 1.0382e-14, 2.2833e-14,\n 6.3089e-14, 8.5694e-15, 2.4632e-13, 1.8225e-13, 2.9055e-13,\n 9.9626e-18, 1.0097e-13, -1.2312e-13, 1.7992e-13, 1.4330e-13,\n 2.6579e-14, -1.1928e-14, -8.5558e-15, 5.4456e-14, 1.3231e-13,\n 2.1632e-13, 2.1866e-13, -4.3861e-14, 4.3482e-13, 4.6059e-14,\n 1.1494e-13, -6.3562e-15, 7.3800e-15, 1.8851e-13, -7.5648e-15,\n 2.4092e-13, 6.9683e-14, 8.8988e-14, 2.1543e-14, 1.3199e-14,\n -2.4021e-14, 1.6327e-13, -5.2960e-15, 1.0206e-13, 1.8680e-13,\n 2.4345e-13, 1.5153e-13, 1.9066e-13, 3.7542e-14, 1.7454e-13,\n 1.4448e-13, 1.1804e-13, 5.4751e-14, 3.9986e-14, 6.8535e-15,\n 2.7989e-13, 3.4407e-14, 2.7293e-15, -1.3613e-14, 5.2156e-14,\n 3.0157e-14, 1.0455e-13, -2.0369e-14, -3.1460e-14, 2.2343e-14,\n 7.9892e-15, -3.9784e-14, -1.8298e-14, 5.3208e-14, 3.8860e-14,\n -6.4966e-16, 2.4552e-13, 4.5682e-14, 2.7749e-13, -2.3241e-15,\n 2.4594e-14, 5.7812e-15, -7.0885e-14, -1.0925e-16, 3.5215e-15,\n 1.0452e-13, 1.1432e-13, 8.8851e-14, -7.9247e-15, 9.2382e-15,\n 3.2287e-14, 3.3001e-14, 1.2642e-13, 5.7902e-15, 5.4964e-14,\n 1.2504e-13, -2.9906e-14, 9.8717e-14, -8.5365e-15, 1.8216e-13,\n -1.3135e-13, 1.7016e-13, -6.6847e-14, 3.4384e-13, -5.1264e-15,\n 9.8061e-14, 2.4134e-13, -1.8631e-15, 8.2223e-14, 1.4820e-13,\n 2.1623e-14, -2.9406e-15, 1.3896e-13, 1.1845e-13, 7.6931e-14,\n 4.9584e-14, 2.2160e-13, -5.4809e-15, 2.2809e-14, -4.9248e-15,\n 1.8834e-13, 1.6935e-14, 2.1519e-14, 3.1167e-13, 1.2360e-14,\n 8.9875e-14, 2.1082e-13, -7.7731e-16, 1.5865e-13, 3.0491e-14,\n -2.0088e-15, 6.4564e-15, -3.7927e-14, -2.0232e-14, 7.7680e-14,\n 3.3434e-13, 1.2500e-14, 2.3373e-14, -1.0501e-15, 6.3524e-14,\n -1.7333e-14, 2.5782e-13, 1.0231e-13, 3.2510e-14, 2.1760e-13,\n 1.4162e-15, -7.5241e-14, 1.2045e-13, 9.9524e-14, -2.5205e-14,\n -4.3746e-14, 1.5462e-15, 1.1582e-13, 3.2523e-14, 6.8270e-15,\n 2.6747e-14, -2.3284e-14, 1.7434e-13, 2.3013e-13, 4.5284e-14,\n 4.8333e-15, -1.0458e-14, 3.2769e-13, 7.6823e-14, 3.1567e-13,\n 2.5636e-13, 1.4558e-13, 1.5474e-14, 1.4853e-13, -1.4703e-14,\n 6.6219e-15, -6.3463e-14, 9.4788e-14, -3.5120e-14, 1.2167e-13,\n -2.6771e-14, -6.9902e-15, 3.4400e-14, 4.2088e-14, 7.8942e-14,\n 8.7443e-14, 5.2411e-15, -1.2715e-15, -2.1161e-13, 8.5781e-14,\n -1.4938e-14, 1.6021e-14, 3.9899e-15, 1.8902e-13, -1.7534e-14,\n 4.7420e-15, 3.1395e-14, 5.2180e-15, -9.8408e-14, -1.5883e-14,\n -3.7381e-14, -1.0140e-14, -4.1254e-14, 6.9246e-14, 7.2037e-14,\n -5.6105e-15, 4.1779e-14, 1.1324e-14, -6.4833e-15, 2.3287e-13,\n -6.4521e-15, 1.6538e-13, 1.0948e-13, 3.6377e-13, 3.5717e-14,\n 2.0469e-13, 2.9365e-14, 2.5076e-13, -6.1042e-14, 3.3457e-14,\n -1.8549e-14, 1.7969e-13, 1.3639e-14, -7.9168e-15, 2.3578e-13,\n 2.2405e-13, 3.2472e-13, 3.8953e-13, 6.9049e-14, -5.7739e-14,\n 3.3424e-13, 2.4579e-13, 1.2395e-14, 3.1762e-15, 4.0520e-13,\n 1.9123e-14, 1.5808e-13, 3.1749e-14, -4.0557e-14, 1.1444e-13,\n 1.3115e-13, 2.0492e-13, -1.4473e-14, 5.1034e-14, -2.4150e-15,\n -4.5865e-15, 1.4024e-13, 2.4127e-13, 2.0067e-14, 7.1627e-14,\n 8.1955e-14, 1.7889e-13, 3.9184e-14, 1.4064e-14, 1.1631e-13,\n 6.6954e-14, 1.6698e-13, 2.2265e-16, -5.8207e-16, -1.7101e-14,\n 1.7485e-13, 2.4276e-13, 1.6530e-14, -8.4238e-14, 2.8211e-13,\n 2.3336e-13, 6.4814e-14, 3.6932e-14, -5.0537e-15, 4.9964e-14,\n 1.7165e-15, 4.0828e-13, 2.1843e-13, 1.2617e-13, 8.3902e-14,\n 7.3162e-14, 2.6770e-14, -4.7480e-14, 2.4869e-14, 4.9321e-15,\n 2.4181e-14, 7.7509e-14, 2.0865e-13, -3.3799e-16, -1.7801e-14,\n 2.3704e-14, 1.4818e-13, 2.1108e-13, -6.0362e-15, 2.4636e-14,\n 1.4923e-13, 3.0139e-14, -7.0385e-14, 7.4912e-14, 2.1241e-13,\n -4.6523e-14, 1.5729e-13, 3.0963e-14, -1.0588e-16, 7.8099e-14,\n 4.2240e-14, 7.1915e-14, 4.4688e-14, -6.8313e-15, 3.1778e-13,\n 1.3956e-14, 4.3124e-14, -2.8356e-15, 4.2795e-14, 6.5359e-14,\n 2.5221e-13, 2.9952e-13, -1.0095e-14, 2.2574e-14, 3.6776e-14,\n 7.6380e-14, 4.1316e-13, 3.7341e-15, 1.3259e-13, 2.6946e-14,\n 2.0891e-14, -2.7535e-14, 9.3921e-15, 9.1678e-15, 2.3283e-15,\n 1.7594e-13, 5.0831e-14, 2.5127e-13, 1.0957e-14, -1.5342e-13,\n 4.9730e-14, 2.5029e-13, 1.7513e-13, 1.8565e-13, 4.2460e-14,\n 1.7368e-13, -1.3847e-15, 8.8997e-14, 9.6444e-14, -5.1666e-14,\n 3.2414e-14, -1.2119e-16, 4.0529e-14, 1.1540e-14, 8.4271e-14,\n 1.8504e-14, -3.1867e-14, 2.2532e-15, 2.7199e-14, 4.6184e-14,\n -3.5359e-15, -1.5373e-14, 1.8926e-13, 5.2309e-14, 4.9481e-14,\n 1.6495e-15, -1.8404e-14, -1.3938e-14, 1.0781e-13, 8.0373e-14,\n -2.4072e-14, 1.6449e-13, 6.0074e-16, 1.2025e-13, 3.3885e-14,\n -6.7150e-15, 3.6385e-16, -2.8131e-15, 9.6853e-15, 7.2980e-16,\n -2.5320e-14, 6.9391e-14, 2.1316e-14, 3.9818e-15, 1.3315e-13,\n 7.2707e-14, -9.2564e-14, 9.2538e-14, 9.5174e-14, 2.6492e-14,\n 1.6045e-13, -5.6370e-14, -2.9689e-14, -2.0631e-14, 4.4761e-13,\n 3.1434e-14, 3.0101e-13, 2.0911e-13, 3.4340e-13, 1.6380e-14,\n -1.7122e-14, 3.2800e-14, -5.6827e-15, 1.0553e-13, 1.8070e-14,\n 3.6424e-14, -1.9892e-14, 4.3505e-14, -7.8501e-14, 2.8862e-16,\n 6.0380e-15, 9.3439e-14, 2.0000e-13, 6.7937e-14, 4.9534e-14,\n 1.2470e-13, 4.0836e-14, 6.0758e-14, 3.3442e-14, 1.2413e-13,\n -3.5646e-15, 1.7608e-13, 5.9546e-15, -1.7867e-14, 1.0989e-14,\n -3.5393e-15, -9.2562e-15, -7.4837e-14, 2.2389e-13, 9.1065e-14,\n -1.1697e-15, 7.8560e-14, -1.8348e-15, 1.3013e-13, -6.7971e-15,\n -5.6637e-15, -4.1128e-15, 8.6068e-14, 2.0828e-15, -4.6231e-15,\n 2.6293e-14, 7.3627e-14, -7.3552e-16, 4.5618e-13, 1.5005e-13,\n 1.3651e-14, -3.0849e-14, 1.4714e-13, 4.1859e-14, -2.1773e-14,\n 2.7870e-14, 1.2561e-13, 5.9133e-14, 1.2289e-13, 4.4696e-15,\n -9.5871e-16, -6.6847e-15, 1.4730e-14, -1.3311e-14, 1.0845e-14,\n 1.6574e-13, 5.4695e-14, 3.1011e-15, 3.9976e-14, -4.5870e-14,\n -6.6820e-15, -7.5493e-15, -2.9482e-15, 1.5325e-13, 1.1303e-13,\n 4.1027e-13, -8.6511e-17, -1.4407e-14, 1.0080e-13, 5.4386e-16,\n -1.8089e-14, 9.4689e-15, 5.6875e-15, 4.8765e-15, -5.3318e-15,\n 1.6800e-13, 7.0356e-14, 3.2861e-14, 2.5550e-13, -4.3428e-14,\n 3.0134e-14, 3.0247e-15, -1.4996e-14, -2.1774e-14, 1.1185e-13,\n 6.0219e-14, -1.8898e-15, 5.3440e-14, 4.0939e-14, 4.3614e-13,\n -2.6538e-15, 2.1504e-14, -1.0231e-14, -3.2255e-14, 9.3709e-15,\n 8.3513e-14, -8.4595e-15, 1.0456e-13, 7.3306e-14, 3.7688e-15,\n 3.2068e-13, 1.3527e-13, 5.9825e-14, 1.1711e-13, 8.7886e-14,\n -1.8727e-15, 6.2690e-14, 1.2215e-13, 2.7130e-13, 3.5750e-14,\n -7.0387e-15, -3.7065e-14, 7.2473e-14, 2.3734e-14, -3.4037e-16,\n -3.5556e-15, 1.4290e-14, 7.4242e-14, 1.4375e-13, 1.1876e-15,\n 1.3850e-13, 5.7987e-15, 2.4986e-14, 4.9373e-14, 2.5010e-13,\n 3.6559e-14, 2.2005e-13, 1.9703e-13, -3.7100e-14, -1.2445e-15,\n -3.6828e-14, 1.7512e-13, 4.7842e-14, 2.2770e-13, 1.8619e-13,\n 5.3836e-14, 8.4425e-15, -3.1459e-14, 2.9699e-13, 5.7644e-14,\n 7.4620e-14, -1.5107e-14, 3.5466e-14, -4.7154e-14, 1.9123e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([8.5142e-18, 3.7978e-17, 2.3130e-15, 1.8831e-15, 5.7013e-16, 3.1112e-15,\n 4.3008e-16, 4.6679e-16, 2.0326e-15, 9.1774e-16, 8.2449e-17, 4.0775e-15,\n 1.4536e-15, 6.2613e-16, 5.2944e-16, 1.8816e-15, 1.9095e-15, 1.5518e-15,\n 7.5816e-17, 3.9084e-16, 3.9755e-16, 9.5224e-18, 5.3136e-15, 3.7638e-15,\n 2.5346e-15, 3.9629e-15, 6.6185e-15, 2.7748e-15, 4.5786e-15, 1.5204e-15,\n 1.6108e-15, 8.5310e-16, 4.5066e-15, 1.5207e-15, 2.7599e-16, 1.4937e-15,\n 3.2071e-17, 6.9325e-16, 1.1622e-14, 6.9175e-17, 1.0620e-16, 1.3199e-15,\n 7.2378e-16, 1.1427e-16, 3.2729e-16, 2.4043e-15, 8.4612e-15, 2.0360e-15,\n 1.8253e-16, 8.0892e-16, 5.9892e-16, 3.0246e-17, 1.9294e-17, 9.6305e-16,\n 1.4237e-16, 1.8981e-16, 1.5751e-15, 9.9328e-16, 1.4864e-16, 1.1659e-17,\n 1.0691e-15, 6.2092e-17, 4.3000e-18, 2.3088e-15, 2.6462e-15, 2.2121e-15,\n 4.0527e-17, 3.9064e-15, 5.4907e-16, 2.6952e-15, 4.2243e-15, 1.4639e-15,\n 4.2764e-15, 3.0785e-15, 3.0092e-15, 1.0578e-15, 3.4185e-16, 5.9850e-16,\n 1.7085e-17, 2.3197e-15, 2.1130e-15, 5.7132e-16, 4.9783e-16, 1.8177e-15,\n 3.8092e-16, 1.7944e-15, 1.4990e-15, 7.3036e-17, 1.5647e-15, 6.9892e-16,\n 1.0713e-15, 1.5366e-15, 6.1556e-15, 2.5050e-16, 1.1200e-15, 4.3303e-16,\n 9.3734e-16, 6.7762e-16, 1.4748e-15, 1.5937e-15, 1.1823e-15, 3.5757e-16,\n 6.2440e-15, 5.0786e-16, 1.4502e-15, 1.8235e-15, 3.9252e-15, 4.1497e-16,\n 4.5656e-15, 3.8227e-17, 5.4659e-15, 3.6076e-16, 3.2940e-16, 1.4612e-15,\n 4.5458e-16, 1.6117e-17, 1.9260e-15, 4.6583e-17, 7.2570e-16, 2.2039e-15,\n 2.4012e-15, 2.6098e-16, 8.2243e-16, 2.2234e-16, 1.2096e-16, 4.7180e-18,\n 2.2136e-15, 2.3536e-15, 4.0535e-15, 1.4064e-15, 1.1435e-15, 2.6814e-16,\n 3.0250e-15, 6.4472e-16, 6.6554e-16, 3.7707e-16, 4.6708e-15, 2.6324e-15,\n 3.1783e-15, 8.6969e-16, 8.7563e-16, 7.5083e-16, 2.1986e-15, 3.2608e-15,\n 8.1941e-17, 3.7253e-15, 4.7655e-15, 1.8901e-15, 8.7630e-16, 7.5501e-16,\n 1.9026e-17, 6.1168e-17, 3.1234e-16, 3.0632e-15, 3.4811e-16, 3.6026e-15,\n 6.9171e-16, 8.0702e-16, 6.5593e-15, 8.6076e-16, 4.9366e-15, 5.6053e-16,\n 5.0307e-16, 2.1541e-15, 7.4792e-15, 5.3398e-15, 2.4784e-16, 3.5465e-15,\n 2.6505e-16, 4.8185e-15, 2.6072e-15, 2.4118e-15, 3.5065e-15, 1.7424e-17,\n 5.8711e-16, 2.2862e-15, 3.3445e-16, 1.8338e-15, 1.2690e-15, 2.2466e-15,\n 5.1325e-18, 5.4609e-16, 1.4344e-16, 5.5154e-15, 4.6752e-16, 1.9137e-15,\n 1.2558e-16, 2.0903e-15, 1.4358e-15, 1.2811e-16, 1.5629e-15, 9.9941e-16,\n 4.1089e-15, 1.8433e-15, 4.4995e-16, 1.0999e-15, 7.6934e-16, 1.3116e-15,\n 2.7054e-16, 3.0175e-15, 9.9052e-16, 3.1301e-15, 3.0922e-15, 5.0659e-16,\n 1.9811e-16, 3.6985e-15, 2.8270e-15, 4.3040e-15, 4.6961e-15, 1.6747e-17,\n 2.6865e-15, 1.5217e-15, 5.1219e-18, 3.9767e-15, 2.3410e-15, 4.5017e-15,\n 1.0656e-17, 4.1200e-17, 7.5873e-16, 8.5473e-16, 2.6885e-15, 8.1387e-16,\n 2.2451e-15, 8.9627e-15, 8.7821e-16, 6.9229e-15, 2.7829e-15, 1.5016e-15,\n 5.2215e-15, 1.2689e-16, 2.7656e-15, 1.2604e-14, 8.7247e-16, 2.1631e-15,\n 2.7770e-16, 4.3059e-16, 1.9763e-15, 1.6012e-15, 1.3605e-17, 2.0353e-15,\n 1.7026e-15, 2.1715e-16, 5.0159e-16, 1.3918e-18, 7.2643e-16, 1.1155e-16,\n 1.5848e-16, 1.9321e-15, 2.3864e-15, 3.7765e-15, 7.1266e-18, 1.1776e-15,\n 1.3944e-15, 7.4920e-15, 1.7662e-17, 9.5262e-16, 2.6780e-15, 3.1311e-15,\n 3.5241e-16, 1.5372e-15, 6.9112e-16, 7.6552e-16, 1.4150e-15, 1.1993e-14,\n 1.9707e-15, 1.8045e-15, 1.9194e-16, 1.2785e-15, 9.6780e-16, 1.5621e-15,\n 2.2416e-15, 3.3097e-16, 5.0980e-16, 4.0219e-15, 3.7046e-17, 9.0868e-16,\n 2.6856e-15, 4.0453e-15, 1.8007e-15, 3.8072e-16, 3.7837e-17, 1.9991e-16,\n 9.2543e-16, 7.9860e-16, 1.5040e-14, 2.8152e-15, 2.2456e-15, 2.5957e-15,\n 5.7241e-15, 3.7034e-15, 1.7920e-15, 3.1119e-15, 1.0863e-16, 3.5575e-16,\n 1.8161e-17, 3.5236e-18, 1.4203e-15, 3.3530e-15, 7.7950e-16, 5.2228e-16,\n 7.7940e-16, 4.2802e-16, 3.8178e-15, 7.5927e-15, 3.0614e-15, 7.8559e-18,\n 8.0151e-15, 1.6611e-16, 1.5954e-18, 4.5330e-15, 5.3901e-15, 3.9784e-16,\n 1.3530e-15, 1.1945e-14, 3.7367e-15, 5.9794e-15, 2.3776e-15, 1.1705e-15,\n 1.0824e-15, 1.5890e-15, 3.5137e-16, 3.9256e-16, 1.9490e-15, 2.8097e-15,\n 1.3372e-15, 3.1694e-16, 3.4187e-15, 5.0311e-15, 4.5152e-16, 7.4064e-17,\n 1.7456e-15, 2.0374e-16, 7.8624e-16, 5.2698e-17, 2.7704e-15, 1.7076e-16,\n 6.8203e-15, 3.1715e-15, 1.2947e-15, 3.4288e-15, 1.0278e-15, 1.4283e-15,\n 5.2798e-17, 1.1060e-15, 1.2054e-15, 6.2394e-16, 3.4392e-15, 6.0478e-15,\n 5.8139e-17, 5.2998e-15, 2.5522e-16, 1.3178e-15, 1.0528e-15, 9.8539e-16,\n 1.2312e-15, 2.5371e-15, 1.0118e-15, 2.1074e-15, 3.0346e-18, 5.8923e-17,\n 2.1828e-15, 1.9369e-15, 2.4963e-16, 1.6687e-16, 4.0821e-16, 2.7632e-15,\n 7.5512e-16, 4.2136e-15, 1.1581e-15, 1.1540e-16, 4.0205e-17, 4.2707e-15,\n 2.2451e-15, 4.1105e-15, 2.1262e-16, 2.4866e-15, 1.6819e-15, 3.0117e-15,\n 3.4744e-16, 1.2322e-15, 5.4568e-16, 2.7951e-15, 5.8311e-16, 2.2421e-15,\n 4.4668e-16, 5.2371e-15, 3.2264e-15, 2.2782e-15, 4.7141e-17, 1.3521e-15,\n 5.5118e-17, 2.2021e-15, 4.2655e-16, 2.6569e-15, 6.2260e-16, 1.5779e-15,\n 1.8131e-15, 8.9488e-15, 1.6356e-18, 2.1097e-15, 3.4549e-16, 1.2913e-15,\n 5.4782e-16, 4.2647e-15, 3.7385e-15, 1.2130e-16, 2.4679e-15, 2.8961e-17,\n 9.7345e-16, 1.7041e-15, 5.4141e-16, 2.4141e-15, 7.4124e-18, 7.6354e-15,\n 1.7460e-17, 1.8693e-15, 3.0989e-15, 2.7430e-15, 3.2226e-15, 1.2575e-15,\n 9.9899e-16, 3.1807e-17, 3.8678e-17, 1.0734e-15, 6.9042e-17, 5.8211e-16,\n 3.3887e-17, 5.8556e-15, 1.2397e-15, 1.4224e-14, 5.5392e-15, 2.7765e-15,\n 2.3931e-15, 7.0997e-15, 2.4904e-15, 1.3006e-15, 2.2581e-15, 1.1690e-15,\n 1.3918e-15, 3.7697e-15, 3.2698e-17, 2.4921e-15, 4.9157e-15, 2.8425e-15,\n 3.6028e-15, 1.2329e-15, 6.8991e-17, 6.4725e-17, 2.0513e-15, 1.3015e-15,\n 2.3935e-17, 1.1177e-16, 4.1709e-15, 3.1105e-17, 4.2433e-15, 3.1118e-15,\n 1.4670e-17, 1.8151e-17, 1.1728e-15, 7.0898e-15, 1.1894e-16, 3.9646e-15,\n 3.2782e-17, 3.7186e-15, 3.1232e-15, 5.0759e-16, 1.6607e-15, 3.2844e-17,\n 4.8040e-16, 4.5084e-16, 2.3664e-15, 5.7422e-16, 1.0037e-16, 1.0685e-15,\n 4.0482e-15, 1.9216e-15, 2.0942e-15, 2.0272e-15, 4.7151e-15, 5.0455e-16,\n 2.0294e-15, 6.5848e-15, 3.1365e-16, 5.9652e-16, 3.7152e-16, 3.5891e-15,\n 1.3751e-15, 1.1677e-15, 1.5543e-15, 1.0930e-15, 2.5946e-15, 2.0840e-15,\n 2.0078e-15, 6.6011e-15, 2.5547e-15, 3.6104e-16, 1.7558e-15, 2.4547e-15,\n 3.2221e-15, 3.6728e-18, 3.2098e-15, 1.8232e-15, 4.5052e-16, 5.4292e-16,\n 3.8575e-16, 9.8173e-16, 1.7544e-15, 3.5777e-15, 1.7404e-17, 7.9241e-16,\n 5.8942e-17, 1.2479e-15, 1.7847e-15, 6.0226e-16, 3.3857e-15, 1.0527e-16,\n 3.2162e-16, 2.6917e-15, 2.1926e-15, 4.8555e-15, 5.3159e-18, 1.6684e-16,\n 1.2172e-15, 6.4313e-16, 2.8915e-15, 2.5757e-17, 1.3091e-15, 7.1671e-15,\n 3.3617e-15, 1.9353e-16, 2.8301e-17, 1.4707e-15, 5.5021e-17, 1.3594e-14,\n 1.4136e-15, 1.6673e-16, 2.9106e-17, 3.1751e-15, 1.3703e-15, 2.2946e-16,\n 1.3033e-15, 5.4774e-16, 1.4349e-16, 1.5537e-15, 2.0357e-17, 3.5042e-18,\n 1.0799e-16, 4.0952e-17, 5.5460e-16, 4.9568e-16, 3.0712e-15, 2.1301e-15,\n 6.7584e-16, 1.2291e-15, 3.5896e-15, 7.3163e-18, 6.6870e-17, 2.4698e-17,\n 2.4935e-15, 4.3779e-15, 3.7168e-16, 1.5243e-17, 4.0616e-17, 3.9318e-15,\n 5.9281e-16, 1.2452e-15, 5.6558e-17, 1.3377e-16, 3.2209e-17, 3.9544e-17,\n 1.3165e-15, 6.3371e-16, 9.5218e-15, 4.6761e-15, 1.7947e-15, 5.4490e-16,\n 1.2482e-17, 2.1553e-15, 3.7248e-16, 2.0328e-15, 2.4290e-16, 3.0589e-18,\n 7.8000e-16, 1.1330e-15, 4.8651e-15, 9.1606e-16, 1.3139e-15, 4.8683e-17,\n 1.1791e-14, 2.0170e-17, 6.3817e-16, 2.2895e-16, 5.1952e-17, 7.5344e-16,\n 4.9988e-16, 2.5775e-16, 2.4362e-15, 2.4410e-15, 1.5498e-15, 1.3107e-15,\n 1.2725e-17, 2.7453e-15, 3.6264e-17, 1.6457e-15, 1.3245e-14, 2.0326e-16,\n 5.5213e-16, 1.7349e-15, 2.5594e-15, 6.2685e-16, 7.4580e-16, 6.0417e-16,\n 4.2737e-15, 2.2501e-15, 8.3977e-16, 3.1141e-15, 5.7583e-17, 1.0382e-15,\n 5.5970e-15, 8.2776e-16, 3.1237e-15, 9.3473e-17, 5.2416e-16, 1.9955e-17,\n 2.2558e-16, 1.7920e-15, 2.8337e-15, 1.2345e-15, 1.0576e-14, 1.0120e-15,\n 4.7488e-15, 9.1078e-16, 1.3914e-15, 5.1908e-16, 1.5222e-15, 3.4173e-15,\n 2.5218e-16, 4.6803e-16, 1.7553e-15, 5.5711e-15], device='cuda:0')" + }, + "58": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-5.6052e-45, -1.1845e-15, 5.8364e-17, ..., 2.9914e-15,\n 3.6700e-15, 8.5845e-17],\n [ 5.6052e-45, -1.8745e-16, 2.4503e-17, ..., 3.2179e-15,\n 1.4271e-15, 1.6972e-17],\n [-5.6052e-45, -1.0046e-15, 1.2462e-16, ..., 2.3474e-15,\n 1.7765e-15, 8.6205e-17],\n ...,\n [ 5.6052e-45, -8.2233e-17, -4.3451e-17, ..., -6.9443e-16,\n -4.1360e-16, -2.5201e-18],\n [ 5.6052e-45, 6.5314e-16, -2.4342e-18, ..., -2.8688e-15,\n -2.9988e-15, -4.3432e-17],\n [ 5.6052e-45, -4.2594e-16, 1.5693e-17, ..., -9.1460e-16,\n -4.9130e-16, -3.5705e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.7953e-16, 1.3362e-15, 1.6485e-15, ..., 6.2827e-18, 1.1860e-16,\n 3.2001e-16],\n [4.5504e-15, 7.8771e-15, 9.7463e-15, ..., 1.1713e-17, 4.9440e-16,\n 8.2069e-15],\n [4.0225e-15, 7.3770e-15, 9.4018e-15, ..., 1.6013e-17, 6.2646e-16,\n 5.6659e-15],\n ...,\n [2.2939e-17, 3.5358e-17, 3.0615e-17, ..., 4.6438e-19, 5.7902e-18,\n 2.5207e-17],\n [1.4702e-15, 3.7263e-15, 3.4237e-15, ..., 2.1401e-17, 2.6667e-16,\n 2.1050e-15],\n [3.8731e-15, 1.4176e-14, 2.0714e-14, ..., 3.3624e-17, 5.6567e-16,\n 9.9942e-15]], device='cuda:0')" + }, + "59": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 3.5856e-12, 1.9264e-12, 2.1122e-12, 1.4372e-14, 1.0927e-12,\n -2.2786e-12, -1.7332e-12, 5.9806e-12, 1.4040e-12, -1.2818e-12,\n -2.7049e-14, -6.8034e-12, 3.4227e-12, 9.7272e-14, 2.8710e-12,\n 5.0972e-12, -1.5208e-12, -2.4266e-12, 3.1019e-12, -1.0557e-12,\n -1.4145e-12, 1.7203e-13, -3.1517e-12, -3.1603e-12, 1.8901e-12,\n -1.1248e-12, -7.1599e-13, 2.6273e-12, 2.1919e-12, 1.2575e-12,\n 4.2269e-13, 4.1122e-12, -2.3282e-12, 2.9198e-13, 3.1380e-12,\n -4.5662e-12, 1.1021e-12, 1.7327e-12, 3.0693e-12, 5.9660e-12,\n 3.3900e-12, -3.0722e-13, -1.3463e-12, -3.9536e-12, 1.7035e-12,\n 2.9019e-13, -5.4766e-14, 3.8746e-12, -1.7690e-12, -1.3158e-12,\n 3.2273e-12, -5.4667e-13, 4.0740e-13, 1.2051e-12, -1.4575e-12,\n -1.4384e-12, 4.0804e-12, 2.7175e-12, 7.2160e-13, 5.9401e-12,\n -5.9928e-12, 4.5011e-13, 1.9121e-12, -1.0821e-12, 1.9810e-12,\n -6.6958e-13, -3.8888e-12, -1.7930e-12, -3.9974e-12, -7.1738e-13,\n 5.7249e-12, 5.0777e-12, -2.9900e-12, 2.0602e-14, 2.4057e-12,\n -2.9082e-12, 3.7236e-12, -7.2118e-13, 2.7453e-12, -2.2576e-13,\n -1.9784e-12, 3.3795e-12, 3.8262e-12, 1.5842e-12, -1.6925e-12,\n 1.9654e-12, 4.0007e-12, -1.1012e-12, 1.2734e-12, -5.6960e-13,\n 1.3942e-13, -7.5933e-13, -8.6606e-12, -6.5841e-12, -1.4020e-12,\n 2.3650e-12, -1.5477e-12, -2.0749e-12, 4.1425e-15, -7.2400e-13,\n -2.8469e-12, -2.9023e-12, -2.5799e-13, 6.8056e-13, -1.0022e-12,\n -1.8553e-12, -7.7000e-13, -7.5761e-13, 6.1270e-12, 4.0794e-13,\n -7.9271e-14, -9.5081e-13, -1.9883e-13, -9.3911e-13, 4.7267e-13,\n -5.2600e-12, -8.3432e-13, 1.1212e-12, -1.3654e-12, 1.1662e-12,\n 4.5740e-12, -2.1820e-12, 1.9987e-13, 2.3274e-12, 1.1805e-12,\n -2.8257e-12, -2.9328e-14, 1.6068e-12, -4.1223e-14, -1.2766e-12,\n -9.2746e-13, -1.4369e-12, -2.0623e-12, 1.5980e-13, 1.2589e-12,\n -1.6250e-12, 1.9261e-12, -3.1104e-12, 4.6379e-12, 5.4581e-13,\n 4.7019e-12, -1.2865e-12, 1.3545e-12, -1.4578e-12, -1.3289e-12,\n -1.4723e-13, 1.2808e-13, 2.8717e-14, 6.3143e-12, -1.2882e-12,\n 8.5787e-13, 2.2794e-12, 7.2566e-12, -9.1958e-13, 6.1126e-12,\n -1.3989e-12, 4.1477e-12, -1.3171e-12, -1.7201e-12, -2.2701e-13,\n 1.9555e-12, 2.6571e-12, -4.4568e-13, 5.4153e-12, -9.2327e-13,\n -1.5313e-12, 6.0847e-13, -1.3786e-12, -2.3554e-13, 2.0376e-12,\n 4.0505e-12, -7.5838e-14, -6.6090e-12, -8.7992e-14, 6.7260e-12,\n -1.2357e-12, -7.1229e-13, -3.5342e-12, -2.9916e-12, -2.6229e-12,\n 2.2822e-12, 7.9109e-12, 2.8146e-12, 6.5506e-13, 1.9358e-13,\n -7.4127e-13, -1.2345e-12, -4.3326e-12, 7.3268e-13, -4.6957e-12,\n -3.5770e-13, 5.2239e-13, 1.2573e-13, -3.0531e-12, -3.4988e-12,\n -1.3491e-12, -1.0775e-12, 3.4668e-13, -2.9271e-12, 6.1780e-13,\n 9.6258e-13, -3.4382e-12, 6.5031e-13, -4.5383e-12, -1.5173e-12,\n -1.2391e-11, 7.4072e-14, 5.1071e-13, -1.0214e-12, -1.0731e-12,\n 1.9904e-12, -9.2768e-13, -5.7591e-13, -3.3403e-12, 2.3097e-12,\n 1.6175e-13, 1.3576e-12, 2.1429e-12, 4.8846e-13, 2.0876e-12,\n -3.7205e-12, -6.8947e-13, -1.5259e-13, -1.9290e-12, -1.8039e-12,\n -1.2785e-12, -5.2660e-14, -1.9311e-12, -1.7397e-12, 3.9429e-12,\n 2.7203e-12, 8.1461e-13, -1.6753e-12, 5.1639e-12, -3.3468e-12,\n 7.3952e-13, -2.1445e-12, -6.1395e-12, 1.2794e-12, -7.0411e-13,\n -1.4998e-12, -9.2726e-13, -1.7617e-12, -1.0134e-12, 2.9975e-12,\n 2.8981e-12, 1.8732e-12, 6.0231e-13, 4.2947e-12, 2.6463e-12,\n -1.8784e-12, -7.1203e-12, 4.8140e-12, 3.1881e-12, 1.7490e-13,\n 2.8489e-12, -5.0164e-13, -1.4169e-12, -2.8351e-12, 1.1597e-12,\n -5.0912e-12, 8.3507e-13, 2.5800e-12, -7.9189e-13, -9.0944e-13,\n -4.7125e-12, -1.9085e-12, -4.8542e-13, -1.2347e-12, 4.4343e-13,\n -1.3637e-12, -6.0155e-13, -4.3134e-12, -9.6245e-13, -1.5331e-12,\n 6.9103e-13, -2.3751e-12, -1.1644e-12, -4.0234e-12, 1.0956e-12,\n 7.5951e-13, -1.1316e-12, -1.0565e-12, -1.6566e-12, 3.1831e-12,\n 5.2264e-12, -3.2039e-12, 9.6747e-13, -9.3211e-12, 3.7603e-12,\n -1.7965e-12, -2.9627e-12, -3.2794e-12, 8.4290e-14, -1.1185e-12,\n -4.8233e-12, -2.4127e-13, -1.3007e-12, -4.1761e-12, -1.2878e-12,\n -6.1989e-13, -1.6063e-12, -6.4083e-12, -7.4438e-13, -1.3324e-12,\n 4.6186e-12, 1.5449e-12, 1.0449e-12, -8.6203e-13, 5.2945e-12,\n 3.7927e-12, 3.1464e-13, -1.4359e-12, -1.1122e-12, -2.3198e-13,\n 2.4685e-12, -1.7517e-12, 8.7699e-13, -6.9953e-12, -1.7033e-12,\n 1.2962e-12, 2.1031e-12, -2.4096e-12, 5.3286e-13, 4.6153e-12,\n 1.0793e-12, 8.3090e-12, 3.6746e-12, -9.9623e-13, -2.2321e-12,\n 4.5825e-12, 4.9312e-12, -1.2611e-12, -1.8787e-12, -6.3377e-13,\n -1.4519e-12, -1.4888e-12, -1.2147e-12, 9.0756e-12, 3.0127e-13,\n -3.4804e-14, -1.4042e-12, 9.5752e-13, -2.1823e-12, 1.2388e-12,\n -5.1980e-12, 1.7862e-12, 3.7750e-12, -1.2900e-12, 8.6136e-14,\n -2.3241e-12, -9.1307e-13, 6.6084e-13, -2.6518e-13, 4.0882e-12,\n -1.2366e-12, -3.1855e-12, -4.5887e-12, -1.6316e-12, 1.0927e-12,\n -6.6522e-13, 2.6621e-12, -6.7028e-13, -8.5275e-12, 9.4818e-12,\n 5.5398e-12, 5.7560e-12, -2.2548e-13, -9.3905e-13, 1.4121e-12,\n -1.0554e-12, 1.0415e-12, 1.3942e-15, 3.2798e-12, -3.5117e-12,\n -1.5875e-12, -1.2910e-12, 1.1395e-11, 4.6110e-13, -1.2918e-12,\n -9.8841e-13, -4.9589e-13, -1.1480e-12, -1.7957e-12, -8.9995e-13,\n -5.2528e-12, -2.8639e-13, -1.3632e-12, 4.7082e-13, 1.0603e-12,\n 5.9253e-13, -9.6148e-13, -6.1384e-13, -2.0591e-12, 4.8051e-13,\n 2.0179e-12, 3.3047e-12, 6.8923e-13, -1.5031e-12, 2.3090e-12,\n 1.2039e-12, -2.3244e-12, -3.7646e-13, -2.2949e-12, -2.2562e-12,\n -1.1783e-12, -1.9631e-12, -1.7948e-12, -2.4159e-12, 5.3312e-12,\n -1.1512e-13, -8.9788e-13, -6.3719e-13, -3.0991e-12, 3.1312e-12,\n -1.8213e-12, 9.5340e-12, 1.6923e-13, 6.0594e-13, -7.2586e-13,\n -1.2611e-12, 4.2416e-12, 4.0171e-12, 6.6166e-12, 7.6485e-12,\n 1.1107e-12, 1.7861e-12, 3.7888e-13, -1.2399e-12, -3.8387e-13,\n -1.1290e-13, 3.0203e-12, 1.4717e-12, 1.8149e-12, -5.3593e-14,\n 2.4796e-12, -1.2692e-12, 7.9050e-12, 9.3866e-13, -2.7831e-12,\n -3.3301e-13, -4.6249e-12, -1.3270e-12, -1.6322e-12, -1.6969e-12,\n -2.9772e-12, 1.6579e-12, -1.3541e-12, 5.1344e-12, 2.4153e-12,\n -9.3265e-13, -3.5394e-13, 3.9944e-13, -1.2944e-12, -2.0704e-12,\n -2.3421e-12, -7.0193e-13, 7.7794e-13, -1.6907e-12, -1.3367e-12,\n 2.2807e-12, 2.3117e-12, -3.3709e-12, 1.3639e-12, -4.6919e-15,\n -4.9056e-12, -3.8289e-12, -1.1735e-12, -3.8674e-12, -2.4913e-12,\n 5.5877e-13, 1.4481e-13, -1.9794e-12, -2.4124e-12, -2.0630e-12,\n -5.0745e-12, -5.8438e-12, -1.9883e-12, -1.3232e-12, -1.7677e-12,\n 5.0447e-13, -2.4503e-12, 8.3798e-12, -9.6673e-13, 9.2788e-12,\n -2.2789e-12, 9.0312e-12, -2.3951e-13, -9.9617e-14, -9.8828e-13,\n -2.5057e-12, 8.7417e-12, -1.6560e-12, 3.5392e-13, -1.2201e-13,\n 1.2738e-12, -5.7158e-12, -3.1073e-12, -5.3948e-12, -1.1542e-12,\n -2.8187e-12, -1.5323e-12, 4.1147e-12, -1.3946e-12, -6.5671e-13,\n -2.3446e-12, -1.4947e-12, -1.9009e-12, -1.3004e-12, 9.2875e-13,\n -3.1958e-12, 7.0058e-13, -1.3170e-12, -8.4011e-12, 2.0247e-12,\n -7.1405e-13, 1.1113e-12, -6.6874e-12, -3.4538e-13, 1.6548e-12,\n 2.7744e-12, -7.8885e-13, -1.0395e-12, -2.5838e-13, -1.2824e-12,\n -6.4449e-13, -3.9980e-12, 1.0410e-12, 1.4806e-12, -7.5203e-13,\n -9.7615e-13, 3.0135e-12, -4.5246e-13, 8.3399e-12, 1.4412e-12,\n 2.9709e-12, 2.3498e-12, 3.1389e-12, -2.7679e-12, 1.5102e-12,\n -3.6419e-13, 1.4070e-12, -7.9391e-13, -3.3315e-13, -2.0124e-12,\n 1.7729e-12, 1.3151e-12, -2.1127e-13, 2.5352e-12, -1.3664e-12,\n 1.6571e-12, 3.3342e-12, -1.3186e-12, -9.1002e-13, -3.8347e-12,\n 7.1723e-14, -1.4084e-12, -1.5429e-13, -3.4628e-12, 3.4324e-12,\n 1.3562e-11, -1.8808e-12, -1.3463e-12, -1.3039e-12, -1.7659e-12,\n 2.2106e-13, 2.2124e-12, -3.3570e-13, 2.1092e-12, -9.2401e-13,\n 5.9151e-12, 8.5096e-13, 5.2793e-12, 2.0020e-12, -4.0138e-12,\n -7.6910e-13, -3.0303e-12, -1.1309e-12, 4.7550e-12, 1.9902e-12,\n -5.0989e-12, -1.8817e-12, 4.8676e-12, 5.2551e-13, 4.3255e-12,\n -8.4885e-13, -1.7909e-12, -8.9408e-13, -1.6162e-12, -1.5214e-12,\n -3.1650e-13, 5.0456e-12, 1.6037e-12, -4.6527e-12, -5.0822e-13,\n -6.8561e-13, -1.1036e-12, -1.6963e-12, 3.9616e-12, -1.1622e-12,\n -1.4272e-12, 1.0014e-12, -6.1474e-13, -1.0194e-12, -4.5263e-12,\n -1.1003e-12, -3.3025e-12, 1.7751e-12, -1.5803e-12, 8.9755e-13,\n 2.2567e-12, 1.4251e-12, -2.1449e-12, -4.5037e-13, -3.6866e-12,\n -7.3342e-13, 2.2071e-12, -1.2612e-12, -2.4666e-12, 7.6033e-12,\n -3.3771e-12, -1.4489e-12, 4.0079e-12, 1.2253e-11, -1.6138e-12,\n -2.8814e-12, -1.9105e-12, -5.7316e-13, 1.8665e-12, -1.0635e-12,\n 5.4215e-12, -1.0262e-12, -9.7091e-13, 7.0443e-12, -8.4884e-13,\n 1.7609e-12, -3.3709e-12, -1.4887e-12, -8.9991e-12, 2.1081e-12],\n device='cuda:0')", + "exp_avg_sq": "tensor([5.3950e-12, 5.7777e-12, 8.7757e-12, 1.3289e-12, 7.6826e-13, 2.7962e-12,\n 2.3730e-13, 1.5995e-12, 1.0635e-11, 3.3609e-13, 4.8042e-12, 1.5782e-12,\n 4.3399e-13, 2.7002e-12, 7.3679e-13, 1.5958e-12, 1.4567e-13, 4.4621e-12,\n 5.1292e-12, 1.8327e-13, 3.1605e-12, 1.1544e-13, 5.1937e-12, 7.7649e-12,\n 4.3266e-12, 3.3000e-13, 6.7879e-12, 6.1687e-13, 4.0331e-12, 2.7525e-12,\n 2.2216e-13, 4.7256e-13, 3.7621e-12, 3.4084e-12, 3.9879e-12, 2.2631e-12,\n 3.6374e-12, 1.4117e-13, 1.0448e-11, 1.0422e-11, 2.1782e-12, 2.6690e-13,\n 7.9003e-13, 5.1912e-14, 4.8061e-14, 2.2862e-12, 1.8960e-11, 3.2225e-12,\n 3.9674e-14, 7.0333e-13, 7.0568e-13, 5.4889e-14, 2.1472e-12, 1.5566e-13,\n 3.7383e-13, 1.2215e-13, 1.2793e-13, 1.8262e-12, 5.7926e-12, 4.2398e-13,\n 4.5711e-12, 2.9405e-12, 6.9393e-14, 4.0575e-13, 5.2152e-13, 4.6560e-14,\n 4.1567e-12, 4.2908e-12, 3.3339e-12, 3.7583e-12, 1.0270e-12, 3.0992e-12,\n 2.5905e-12, 2.9810e-12, 9.4093e-12, 1.8843e-13, 7.7254e-13, 1.5787e-12,\n 7.7012e-13, 1.2144e-11, 2.6714e-12, 2.0545e-13, 5.1851e-13, 1.0292e-13,\n 3.0126e-13, 4.3115e-12, 5.3264e-12, 7.2876e-12, 5.5274e-13, 1.1589e-12,\n 1.8262e-12, 4.1337e-12, 3.6392e-12, 2.3409e-12, 6.7382e-13, 2.6074e-13,\n 9.0671e-13, 3.4965e-12, 2.0110e-12, 1.6842e-13, 1.7027e-12, 3.7896e-13,\n 3.0098e-12, 1.9293e-12, 1.6115e-13, 1.0775e-12, 2.5362e-11, 5.8509e-14,\n 4.3560e-13, 2.9363e-13, 1.1296e-12, 3.9442e-13, 9.1506e-14, 4.6399e-12,\n 1.9480e-12, 1.2826e-13, 1.3539e-13, 9.6247e-13, 9.3344e-13, 6.7251e-13,\n 5.4460e-12, 4.1402e-12, 3.2215e-13, 2.7636e-13, 2.2509e-12, 5.0006e-12,\n 4.4863e-13, 2.5090e-13, 1.3307e-13, 2.4743e-12, 8.6918e-13, 1.9687e-12,\n 5.7690e-12, 2.7142e-12, 9.3031e-13, 4.3491e-13, 2.9485e-12, 1.3897e-12,\n 1.4438e-12, 1.0362e-13, 5.7117e-12, 5.1122e-13, 1.1569e-13, 2.3611e-12,\n 9.5778e-12, 1.0969e-13, 6.1811e-13, 3.5722e-12, 2.2934e-12, 9.0109e-14,\n 4.7124e-12, 3.1764e-13, 2.8474e-13, 1.5411e-13, 7.4722e-12, 9.2354e-13,\n 2.1037e-12, 3.9841e-13, 2.6152e-12, 3.6082e-13, 4.6235e-12, 4.1730e-12,\n 1.0088e-12, 1.4154e-13, 9.5441e-12, 1.8856e-12, 1.5197e-12, 1.4830e-12,\n 3.6541e-12, 5.2643e-12, 8.3825e-12, 4.7368e-12, 2.1674e-12, 2.7487e-12,\n 6.1906e-12, 4.3213e-13, 4.7948e-14, 3.1393e-12, 7.2325e-12, 8.8224e-13,\n 1.8489e-12, 2.2242e-12, 1.6508e-12, 5.8567e-12, 1.2491e-11, 6.6304e-12,\n 2.2205e-13, 1.7342e-12, 7.3511e-13, 4.5288e-12, 2.6991e-12, 2.3008e-13,\n 1.8128e-13, 1.4260e-12, 5.3859e-12, 5.0793e-12, 6.0325e-12, 6.1498e-12,\n 4.8596e-12, 4.0880e-12, 1.1219e-12, 5.0256e-12, 2.4090e-13, 1.5847e-12,\n 5.3239e-13, 6.2883e-12, 2.3663e-12, 2.7006e-13, 4.5195e-13, 4.8626e-13,\n 1.3474e-12, 5.6497e-14, 1.2175e-12, 2.2622e-12, 1.6377e-12, 1.5573e-12,\n 6.4951e-13, 4.1814e-12, 1.3434e-12, 1.5486e-11, 6.4763e-13, 3.2040e-14,\n 7.9288e-12, 3.4270e-12, 7.6459e-13, 8.3443e-14, 1.3440e-12, 2.9166e-12,\n 9.9806e-14, 1.1533e-11, 5.4593e-12, 1.8881e-12, 2.5703e-12, 6.4734e-14,\n 3.8468e-12, 1.0459e-13, 1.8726e-12, 2.8040e-12, 5.0788e-12, 3.2920e-13,\n 8.9254e-14, 3.2917e-13, 1.9401e-12, 1.1870e-12, 2.8560e-12, 2.1384e-12,\n 1.1763e-11, 9.4653e-12, 3.8892e-13, 5.5927e-12, 7.0475e-13, 4.9652e-12,\n 2.3191e-12, 1.0465e-11, 4.0625e-12, 8.0654e-12, 1.4062e-13, 6.8303e-12,\n 2.3248e-12, 1.9976e-13, 2.8870e-12, 1.8015e-13, 7.5209e-14, 2.6850e-13,\n 4.7841e-14, 6.8874e-13, 6.9579e-14, 4.9115e-13, 1.1010e-12, 2.0954e-12,\n 4.8689e-13, 5.6417e-14, 1.9326e-12, 4.2196e-12, 7.6533e-13, 8.9250e-12,\n 2.7173e-12, 6.4394e-13, 5.0527e-13, 1.1750e-11, 2.2059e-14, 2.1870e-12,\n 4.7660e-13, 1.2749e-12, 3.4492e-11, 1.1439e-11, 2.6044e-12, 3.1229e-13,\n 2.6139e-12, 2.4066e-13, 3.8302e-12, 2.3669e-12, 2.1608e-12, 1.1731e-13,\n 1.4232e-13, 7.0692e-13, 6.8619e-12, 1.0438e-13, 4.7443e-12, 1.1319e-12,\n 3.4785e-13, 1.9543e-12, 1.2431e-11, 2.9831e-13, 2.9004e-12, 3.4162e-12,\n 5.0887e-13, 2.9984e-13, 9.1620e-14, 3.2027e-13, 2.6551e-13, 2.5352e-12,\n 1.5921e-12, 2.1580e-12, 4.4590e-12, 5.6444e-12, 4.5010e-12, 1.5060e-13,\n 1.7425e-12, 9.8376e-13, 6.2710e-13, 9.2190e-13, 1.0064e-12, 1.0820e-13,\n 2.8282e-13, 5.3036e-13, 2.9692e-12, 1.3849e-13, 6.8898e-12, 1.1673e-12,\n 7.5578e-12, 3.6066e-12, 8.3463e-13, 4.4408e-12, 8.2623e-12, 9.5261e-13,\n 1.9288e-12, 2.3976e-12, 4.4983e-12, 1.1673e-13, 7.2696e-14, 1.8278e-12,\n 7.9945e-12, 9.8212e-13, 2.0187e-13, 3.2463e-12, 1.1395e-13, 8.6756e-12,\n 1.1050e-13, 4.9966e-12, 2.3968e-12, 1.3848e-13, 1.0513e-12, 1.2748e-11,\n 8.3788e-12, 1.1019e-13, 4.8823e-12, 1.6940e-12, 9.5320e-12, 1.3640e-12,\n 2.3322e-13, 4.2554e-12, 3.4805e-12, 5.7050e-12, 2.4749e-12, 3.1192e-12,\n 1.0292e-12, 8.9091e-12, 9.2779e-12, 6.8205e-12, 5.9571e-13, 1.8352e-12,\n 1.8710e-12, 4.8887e-12, 5.3280e-12, 8.5370e-13, 4.7284e-13, 4.5335e-14,\n 2.2595e-11, 5.5503e-14, 6.2882e-14, 2.0399e-12, 1.9587e-12, 4.8288e-12,\n 2.8804e-13, 9.3831e-12, 1.5640e-12, 1.2269e-12, 3.0903e-12, 1.0054e-12,\n 1.8463e-13, 2.4934e-12, 2.3644e-13, 1.3636e-12, 3.9116e-12, 8.2741e-14,\n 1.7508e-12, 2.4592e-13, 3.0286e-14, 5.9149e-13, 3.2187e-12, 3.4360e-12,\n 9.8963e-13, 2.4232e-13, 3.2367e-12, 9.8115e-14, 3.0074e-12, 2.8153e-12,\n 2.2878e-13, 3.0241e-12, 3.3688e-13, 1.4339e-12, 1.6247e-12, 1.5023e-11,\n 2.7539e-13, 1.8047e-12, 4.8541e-12, 1.2177e-12, 7.9577e-13, 8.5598e-13,\n 9.2810e-14, 3.7972e-12, 1.6897e-12, 7.7450e-13, 4.2453e-12, 4.9145e-13,\n 5.6807e-12, 6.5308e-13, 9.1910e-13, 2.6222e-13, 3.7884e-12, 9.9944e-12,\n 2.1518e-12, 7.2259e-12, 3.5822e-13, 7.4769e-12, 6.1863e-14, 6.7076e-13,\n 7.7760e-13, 1.4204e-13, 1.3646e-12, 1.6462e-12, 1.6957e-12, 7.5195e-13,\n 1.5304e-13, 3.6521e-12, 2.8002e-13, 9.2104e-13, 9.8616e-12, 8.3791e-12,\n 4.1115e-12, 4.0894e-14, 4.0951e-12, 1.4418e-12, 6.3157e-12, 1.3593e-12,\n 3.0723e-13, 4.3350e-12, 3.7400e-12, 7.4693e-13, 9.9659e-12, 7.1256e-12,\n 1.0133e-11, 3.0180e-12, 9.0331e-12, 9.2963e-12, 1.6420e-12, 3.8881e-13,\n 8.7862e-14, 9.7584e-13, 2.9910e-13, 2.2443e-13, 1.5080e-12, 4.0358e-13,\n 2.4295e-13, 2.0921e-12, 1.7604e-12, 4.3103e-12, 1.4102e-12, 3.1980e-12,\n 2.1363e-12, 1.0275e-11, 6.5255e-13, 3.2837e-12, 1.5582e-12, 1.8599e-12,\n 1.4910e-11, 1.0458e-12, 4.7888e-13, 1.5058e-12, 1.2741e-11, 3.3816e-11,\n 6.7542e-13, 1.4886e-13, 7.1234e-12, 1.8714e-13, 7.2209e-12, 1.6899e-13,\n 1.7876e-12, 3.2127e-14, 1.2420e-13, 2.7664e-13, 1.1900e-12, 9.3625e-12,\n 4.8078e-12, 5.9354e-12, 1.0108e-12, 1.8658e-12, 4.2211e-13, 1.1954e-12,\n 7.0838e-12, 2.0757e-13, 7.7833e-13, 4.7330e-12, 4.7828e-13, 3.8422e-12,\n 1.0375e-11, 2.9547e-12, 9.4776e-13, 2.4683e-13, 8.5359e-12, 2.4748e-12,\n 6.4828e-12, 1.5621e-12, 4.2551e-12, 9.8913e-12, 8.1282e-14, 1.0659e-11,\n 8.7187e-14, 1.1028e-11, 4.2954e-13, 6.7737e-13, 1.0691e-12, 3.2146e-11,\n 3.0278e-12, 6.1352e-12, 7.5055e-13, 8.9401e-12, 9.1935e-12, 1.5286e-12,\n 2.7867e-12, 2.3891e-12, 1.7977e-12, 1.5756e-12, 1.1063e-12, 4.0957e-13,\n 1.1781e-12, 1.9264e-12, 1.6380e-13, 2.2175e-12, 7.0146e-12, 2.2634e-12,\n 3.3628e-13, 4.4299e-12, 2.1640e-12, 5.2858e-13, 8.9861e-13, 1.7145e-12,\n 8.1465e-12, 5.7821e-12, 2.5468e-11, 2.6589e-13, 2.6722e-12, 1.3721e-12,\n 5.2992e-13, 4.2053e-12, 2.7225e-12, 4.6544e-14, 2.8538e-13, 1.0011e-13,\n 1.4987e-12, 3.8593e-12, 3.1059e-11, 1.2111e-12, 2.2702e-12, 9.6952e-13,\n 1.1127e-12, 1.4977e-12, 5.3055e-12, 8.7421e-13, 5.0974e-13, 4.2241e-12,\n 4.9910e-12, 1.1393e-12, 1.7482e-12, 7.5848e-14, 9.8926e-13, 1.8531e-12,\n 7.0277e-12, 2.6955e-13, 3.2094e-13, 2.4206e-13, 4.0014e-12, 1.3201e-12,\n 3.5887e-12, 8.7790e-13, 5.1278e-14, 1.9429e-13, 3.3406e-12, 2.4976e-12,\n 4.8320e-13, 8.9703e-14, 1.4626e-13, 3.7638e-13, 2.4626e-11, 1.0463e-13,\n 5.5932e-12, 2.7663e-12, 9.9958e-14, 2.2055e-12, 1.5873e-13, 5.4128e-13,\n 4.9795e-12, 2.5217e-13, 3.3451e-13, 1.9033e-13, 4.9345e-12, 5.1320e-13,\n 3.0753e-12, 9.3023e-12, 2.7579e-13, 2.1409e-12, 1.3497e-12, 7.5500e-12,\n 7.7165e-14, 2.1466e-12, 1.3625e-12, 1.5461e-11, 6.9009e-13, 4.5503e-12,\n 6.3015e-12, 5.7245e-13, 3.1692e-13, 1.3317e-12, 1.0707e-13, 2.3190e-12,\n 9.3562e-13, 5.3465e-14, 6.7799e-12, 1.0058e-11], device='cuda:0')" + }, + "60": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 6.0820e-14, 3.2975e-13, 4.9583e-14, 3.9962e-14, 5.2167e-14,\n 3.7867e-15, -3.6885e-17, 4.0780e-13, 4.2060e-14, -1.2949e-15,\n -1.9730e-14, -6.6322e-14, 2.1200e-15, 8.5523e-14, -1.6984e-14,\n 3.1917e-13, 2.4116e-14, 2.4972e-14, 4.8745e-13, -9.3925e-16,\n 4.9778e-14, 2.9647e-14, 5.4895e-14, -1.2619e-15, 2.6132e-14,\n -8.2255e-16, -6.1874e-15, -2.0027e-15, 4.9512e-16, 2.2339e-13,\n -8.3573e-15, 3.4618e-14, 2.2417e-16, -1.7876e-14, 7.3544e-14,\n -1.7564e-14, 1.5033e-14, 1.2462e-15, 4.0968e-14, 4.8560e-13,\n 1.3952e-13, 1.3210e-14, -2.2575e-15, -7.1056e-15, 1.3826e-14,\n 3.3585e-15, 1.3367e-13, 1.5526e-13, -1.8748e-15, -4.8714e-15,\n 7.0059e-14, 3.3587e-15, -1.9217e-14, 2.6476e-13, 2.6283e-14,\n 1.8534e-14, 8.3705e-14, 2.1616e-15, 9.9217e-14, 1.4356e-13,\n -3.3601e-14, 6.5645e-15, 3.1661e-15, 4.9175e-15, 9.9318e-14,\n 4.6497e-15, -1.2367e-14, 2.3364e-14, 4.3110e-14, 8.6175e-14,\n 6.3202e-14, 5.6923e-14, 5.4831e-14, 2.0459e-13, 4.0052e-14,\n 4.4821e-15, 1.1701e-13, 9.9596e-14, 3.0166e-15, 4.5110e-14,\n 2.7022e-14, 1.1867e-13, -2.1456e-14, 1.0372e-14, 9.9006e-15,\n 2.5444e-13, 3.2389e-13, -6.7266e-16, -2.6145e-15, 2.5385e-14,\n -1.4813e-14, -7.0614e-15, -9.9981e-14, -6.1156e-14, 1.2748e-14,\n 3.3089e-13, -4.7864e-15, 4.1185e-16, 1.1422e-13, 1.2960e-14,\n 3.6999e-15, -3.1306e-17, -1.6208e-14, -1.2319e-14, -5.9854e-15,\n -6.5292e-16, 1.4550e-13, 1.7072e-14, 1.1519e-13, 5.9952e-15,\n -1.2004e-15, 2.1169e-15, 1.8617e-15, 8.1903e-14, -9.8524e-15,\n -2.8310e-14, 6.6296e-15, -1.5671e-14, 3.2914e-16, 1.3399e-15,\n 1.0879e-13, 1.9722e-14, -1.5470e-14, 7.0648e-14, -2.5891e-14,\n 3.0365e-15, -3.6686e-16, -9.9211e-15, 1.3332e-14, -1.1552e-15,\n -7.3919e-15, -3.7718e-15, -2.2914e-15, 8.4839e-14, -6.3682e-15,\n 1.3240e-14, 4.5709e-14, -2.6321e-15, 1.2502e-13, 2.9023e-14,\n 2.1524e-13, 7.5628e-15, -5.8930e-15, 5.2409e-15, 3.0534e-16,\n 4.2670e-14, 6.1121e-15, 3.7749e-14, 7.7978e-14, -1.5519e-16,\n 4.0384e-15, 9.2791e-15, 4.3645e-13, 3.3432e-14, 6.8931e-13,\n -5.5297e-15, 1.8919e-14, 5.9243e-15, 9.9311e-15, 1.4461e-14,\n 2.2968e-14, 2.1627e-13, -1.1600e-14, 1.2259e-13, -8.8152e-16,\n 1.5283e-14, 4.2816e-14, 9.2068e-15, -3.2447e-15, 1.9181e-13,\n 1.5322e-13, 3.3544e-14, -5.1519e-14, -7.5231e-15, 9.3341e-15,\n 2.8027e-15, 7.0142e-14, 4.0001e-15, 2.6791e-14, 2.3531e-14,\n 8.8696e-14, 3.0033e-13, 4.6563e-14, 1.1277e-14, 1.0633e-13,\n 7.4666e-16, -2.9983e-15, -1.0957e-14, 9.0777e-15, -1.3336e-14,\n 9.6169e-16, -4.3159e-15, 5.8055e-14, 3.4380e-15, 7.8325e-15,\n -7.7573e-16, 2.5425e-13, 7.4827e-14, 3.1517e-16, -4.0247e-15,\n 1.5376e-13, 9.4832e-14, 5.7135e-15, -1.7707e-14, 4.7132e-14,\n -1.3890e-13, -9.5654e-15, 9.5497e-15, -5.9881e-15, 1.8892e-16,\n -2.3925e-14, -6.4527e-15, 3.4917e-15, 2.1951e-14, 7.9586e-14,\n -2.6001e-14, 4.1060e-14, -3.8096e-14, 1.2470e-13, 6.8116e-14,\n 5.2153e-14, -3.4786e-15, 1.2193e-15, -1.7947e-15, 2.3570e-15,\n 8.2354e-16, -7.5863e-15, 3.2962e-14, 5.3305e-14, 1.5916e-13,\n 3.8338e-14, 1.5787e-14, 1.6533e-14, 7.5166e-14, 4.9991e-14,\n 4.7686e-15, 4.7226e-15, -4.2826e-14, 4.6864e-14, -1.2927e-14,\n 2.8325e-14, -6.6547e-15, -1.5141e-15, 4.5236e-14, 7.4192e-14,\n 2.9565e-13, 7.6132e-14, 1.0282e-13, 1.3444e-13, 5.4213e-14,\n -3.6054e-16, -6.5228e-14, 4.6837e-14, -1.0552e-14, 3.4503e-15,\n 4.5440e-15, -1.0223e-14, 1.9634e-14, 7.4148e-14, -2.9982e-16,\n 8.4070e-14, 3.7808e-15, 5.7223e-15, -6.5908e-15, 3.0070e-14,\n -1.6774e-14, -1.7934e-15, 7.8617e-15, -2.1510e-15, 5.9480e-16,\n 5.2563e-15, -1.5157e-14, -8.4473e-15, 8.3058e-14, -3.0517e-15,\n 3.3867e-14, 5.9453e-15, 1.3048e-14, -6.1109e-15, 6.8758e-14,\n -4.6380e-15, 2.0800e-14, -2.7438e-15, 9.1670e-17, 1.0235e-13,\n 1.4170e-13, -4.9483e-15, -4.8764e-15, -1.5875e-13, 4.0567e-14,\n 2.3728e-14, 2.9306e-15, 1.1696e-13, 3.0147e-14, -1.4239e-15,\n 1.9151e-15, -9.0763e-15, -3.1159e-15, -7.5464e-15, 1.2912e-14,\n -5.1309e-15, 1.0955e-14, -3.5872e-14, 2.8547e-14, -8.3033e-16,\n 1.1828e-13, 6.5666e-15, -3.2276e-14, 5.2743e-14, 4.4129e-13,\n -3.2605e-16, -2.5944e-14, -1.5587e-15, -2.6231e-15, 6.1664e-14,\n 2.4025e-14, -2.3383e-15, -1.5253e-14, -6.6619e-14, 7.1234e-14,\n 3.5973e-13, 5.6386e-14, 2.0977e-14, 1.8077e-14, 1.9385e-13,\n 2.3938e-14, 3.9558e-13, 3.1420e-15, 7.5658e-14, 7.4851e-14,\n 2.5528e-13, 1.1614e-13, 9.5856e-15, 9.9775e-14, -1.4882e-14,\n -5.2825e-15, 3.0333e-14, -5.1610e-15, 2.8277e-13, 2.2907e-14,\n -1.4807e-14, 5.4725e-14, 1.8248e-14, 4.9337e-14, 5.5765e-15,\n -2.3115e-14, 1.9288e-14, 5.7546e-14, 5.9411e-15, -1.0309e-15,\n 9.7520e-14, 2.3591e-14, 4.9110e-14, -7.3853e-15, 1.8392e-13,\n 2.9004e-14, 7.2114e-14, -1.9506e-14, 4.3777e-13, 2.2727e-13,\n 4.5517e-15, 9.9520e-14, -7.3232e-15, -9.5701e-14, 5.9608e-13,\n 5.0087e-14, 2.8907e-13, 1.7462e-13, 8.0459e-15, 4.6374e-14,\n -7.8514e-15, -2.0013e-14, -1.7808e-14, 2.6063e-13, 7.1710e-14,\n -1.4394e-14, -1.1590e-15, 2.9281e-13, 1.4871e-13, 2.6408e-16,\n 8.6073e-15, -6.5747e-15, 4.4556e-15, 8.8118e-15, 6.5433e-15,\n -8.3904e-15, 8.7907e-15, -1.6290e-15, 5.1780e-16, -2.7314e-14,\n 3.0851e-15, 2.2234e-15, -6.3995e-15, 2.5396e-15, -2.7892e-15,\n -1.1438e-15, 3.5437e-13, -3.1374e-16, 9.2007e-15, 6.9276e-14,\n 2.6878e-14, 4.7088e-14, -1.0701e-14, 7.6465e-14, 4.7904e-14,\n 6.2875e-15, 5.7631e-15, 4.6142e-16, -5.3215e-15, 1.7445e-13,\n -1.1098e-14, 3.4398e-15, -9.0962e-16, -1.4041e-15, 3.5579e-16,\n 1.3941e-13, 1.8946e-13, -6.3561e-15, -2.3897e-14, 1.3123e-13,\n -9.1954e-16, 7.6818e-14, 1.6564e-13, 2.0094e-13, 2.3705e-13,\n 5.1070e-15, 2.8663e-13, -1.6262e-14, 1.1706e-14, -1.2388e-14,\n 2.1349e-14, 1.0288e-13, 4.9294e-14, -1.9572e-15, 1.7143e-14,\n 7.4121e-14, -5.6802e-15, 2.7610e-13, 4.9130e-14, -5.6298e-17,\n 1.9169e-15, -8.7467e-15, -3.9321e-15, -9.4925e-16, 4.5045e-14,\n 4.9212e-14, -1.4677e-14, 3.5471e-14, 1.5457e-13, 4.5837e-14,\n 9.6937e-15, 1.1112e-14, 4.7557e-14, 6.0152e-15, 4.9176e-14,\n 2.7522e-15, 4.2518e-14, 7.4483e-14, 2.4760e-14, -2.0595e-15,\n 4.5415e-14, 4.3761e-14, 7.7202e-15, 3.2757e-14, 6.5626e-14,\n -1.8984e-14, -9.1636e-15, -1.3262e-15, -3.1981e-15, 1.1694e-15,\n -9.8758e-15, -7.1280e-15, 9.6420e-15, 2.2940e-15, 1.6075e-15,\n 5.3115e-14, -4.1798e-14, 3.9976e-15, 4.5163e-14, -3.0099e-15,\n 1.0385e-14, 7.9368e-16, 2.5991e-13, -2.5249e-15, 1.5623e-13,\n 1.7580e-14, 1.3456e-13, -1.5753e-14, 6.4632e-15, -4.4888e-15,\n 6.1265e-14, 4.8488e-13, 3.0736e-15, -1.7801e-15, -1.0735e-14,\n 1.9665e-14, -3.5162e-14, 2.8017e-14, -3.5931e-15, 6.6897e-16,\n 5.5316e-15, 7.6991e-15, 3.4739e-14, 4.2162e-15, 1.9161e-13,\n 1.0386e-13, -1.4614e-15, 1.3779e-14, -7.6360e-15, 4.7650e-14,\n -6.8687e-16, 6.4715e-14, -5.4748e-15, -1.1721e-13, 2.3845e-15,\n -4.7726e-15, 7.7653e-14, -7.1260e-14, 1.2376e-13, 1.0804e-14,\n 1.3822e-13, 1.1389e-14, 2.5635e-14, 1.3926e-13, -4.5012e-15,\n 9.0521e-14, -7.4111e-15, 1.5992e-13, 6.8291e-15, -5.4323e-15,\n -3.1884e-15, -1.8568e-14, 3.1810e-15, 6.5002e-14, 1.1759e-14,\n 9.3864e-14, -2.2182e-14, 2.8119e-13, 4.3605e-14, 2.0205e-14,\n -1.0812e-14, 1.4372e-13, -6.1025e-15, 6.7331e-14, 3.8980e-15,\n 3.3203e-15, -1.7419e-15, 1.5448e-14, 1.6989e-13, -5.7462e-16,\n 4.5280e-14, -5.9328e-15, 3.2047e-14, 8.0491e-16, -5.5166e-15,\n 1.8215e-14, -2.4624e-15, -5.0217e-15, 8.6856e-14, 1.0698e-13,\n 2.2652e-13, -4.1293e-16, 9.6150e-15, -6.3829e-15, -2.1437e-15,\n -3.8431e-15, 4.3132e-14, -6.2902e-15, 1.1519e-14, 3.8607e-14,\n 1.9780e-13, 5.3993e-14, -2.6563e-15, 1.6974e-14, -5.1062e-15,\n 1.1329e-13, -2.4404e-15, 1.2512e-14, 1.5136e-13, -1.0792e-14,\n 4.0477e-14, 1.6983e-14, 1.2426e-13, -4.4644e-15, 1.8195e-14,\n -1.0348e-14, 6.4965e-15, -6.2041e-15, 1.2099e-15, 1.3870e-14,\n 9.7476e-15, 1.6318e-13, 1.4033e-14, 8.7770e-14, -2.6721e-16,\n -8.9340e-15, -5.7493e-15, 5.0167e-14, 7.4121e-14, 2.3506e-14,\n -5.1603e-15, -1.5198e-14, -7.4745e-16, -3.8041e-15, -1.7994e-14,\n -5.6763e-15, 1.2041e-15, 4.1569e-13, -1.7539e-15, 3.7671e-14,\n 3.1252e-15, -5.2185e-15, 8.7875e-15, -1.0774e-14, 8.7629e-15,\n 1.4804e-14, 8.7737e-14, -5.2537e-15, 2.6608e-14, 3.0973e-13,\n -4.9151e-15, 5.9527e-15, 1.9862e-13, 4.0622e-13, 6.2979e-15,\n -4.3134e-16, 4.2538e-14, -3.9305e-15, 1.4482e-13, 8.9580e-15,\n 2.3226e-13, -8.0955e-15, -3.3879e-15, 7.0303e-13, -3.8770e-15,\n 1.4899e-14, 5.0039e-15, -4.4962e-15, -1.2348e-13, 1.9340e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([3.5657e-15, 2.5466e-15, 1.0648e-14, 1.2979e-16, 1.3108e-16, 2.1764e-15,\n 1.6464e-17, 1.2496e-15, 2.6342e-14, 1.3558e-17, 3.6627e-15, 1.0897e-15,\n 1.7830e-17, 8.5542e-15, 2.8612e-17, 3.3043e-16, 6.0994e-17, 4.9956e-15,\n 3.0759e-15, 3.7050e-17, 4.8625e-16, 6.9914e-17, 1.2051e-14, 1.1645e-15,\n 5.2876e-16, 8.8729e-18, 9.2060e-15, 1.0997e-17, 1.4035e-15, 6.4570e-16,\n 6.6043e-17, 5.7106e-17, 2.1343e-15, 9.0721e-16, 7.4274e-15, 5.6351e-16,\n 1.6246e-15, 3.1802e-17, 7.0747e-15, 2.6077e-14, 7.7215e-16, 1.9998e-17,\n 1.7801e-16, 3.9707e-17, 1.4014e-17, 5.2570e-16, 3.2147e-15, 1.7484e-15,\n 4.1690e-17, 6.5034e-16, 2.8488e-17, 5.9613e-17, 1.4492e-15, 3.3972e-17,\n 4.8099e-16, 1.1884e-16, 8.5417e-18, 1.6814e-16, 2.2309e-15, 7.4919e-17,\n 1.4694e-14, 1.4893e-15, 1.1862e-16, 1.2994e-16, 8.9273e-17, 7.1637e-18,\n 1.1207e-14, 2.8432e-16, 3.0717e-15, 2.3925e-15, 1.4510e-16, 4.7530e-15,\n 3.0567e-15, 1.2727e-15, 2.8008e-14, 4.6011e-18, 1.0399e-15, 6.2733e-16,\n 1.9266e-17, 8.6038e-15, 3.9478e-15, 9.1335e-18, 5.9633e-17, 8.1564e-17,\n 1.7735e-17, 5.0110e-15, 1.5829e-15, 3.7561e-15, 1.8288e-17, 3.6956e-16,\n 5.0486e-16, 1.9265e-15, 2.7170e-15, 6.9564e-17, 6.8186e-17, 2.3264e-17,\n 1.2093e-16, 1.8479e-15, 1.2577e-15, 4.5123e-17, 3.9628e-16, 2.0609e-17,\n 1.1336e-15, 3.1141e-15, 6.0049e-18, 1.8579e-16, 2.6346e-14, 2.4805e-17,\n 4.7542e-16, 6.1465e-17, 1.8366e-16, 5.8407e-18, 6.9986e-18, 3.9925e-15,\n 7.0181e-16, 5.6956e-18, 6.6963e-17, 1.3384e-17, 2.1745e-17, 2.2755e-17,\n 6.4692e-15, 1.0809e-14, 8.4503e-18, 3.6530e-17, 6.5981e-16, 1.1183e-15,\n 2.9978e-17, 8.7773e-17, 1.0151e-17, 5.2323e-16, 4.7745e-17, 3.6630e-16,\n 3.3296e-15, 4.2561e-15, 1.2588e-16, 2.4617e-17, 1.0203e-15, 8.4777e-17,\n 5.8449e-16, 2.4853e-17, 2.5687e-15, 6.0198e-18, 1.6088e-16, 3.7371e-16,\n 1.1624e-14, 1.2375e-16, 1.9304e-17, 1.0845e-14, 5.7280e-16, 3.9591e-18,\n 1.9520e-15, 5.7795e-17, 2.2934e-17, 5.5021e-17, 4.4671e-15, 2.1005e-16,\n 3.1691e-16, 2.4145e-17, 1.3377e-15, 1.3337e-17, 7.4282e-16, 3.0245e-15,\n 1.9337e-16, 2.3377e-17, 5.4380e-15, 1.9531e-15, 1.2881e-15, 4.8270e-16,\n 2.6178e-16, 3.4459e-15, 5.0229e-15, 6.0396e-16, 1.0797e-16, 7.1589e-16,\n 4.9557e-16, 8.3792e-18, 9.5813e-18, 6.0691e-16, 8.5284e-15, 1.6614e-17,\n 7.9752e-16, 5.1434e-16, 2.7267e-16, 8.9090e-16, 1.7402e-14, 5.8148e-16,\n 1.6798e-18, 2.5662e-16, 8.0671e-17, 5.9466e-15, 1.0447e-15, 3.0454e-17,\n 4.6362e-17, 2.8191e-16, 5.5555e-15, 1.1859e-15, 3.7456e-15, 5.9685e-15,\n 2.8658e-15, 6.9517e-16, 8.4982e-17, 1.1272e-14, 1.0963e-17, 2.6753e-16,\n 1.0706e-16, 3.0898e-15, 4.0456e-16, 4.1281e-17, 1.2108e-17, 2.0325e-17,\n 1.2827e-15, 3.4902e-18, 7.1397e-16, 1.1511e-15, 9.6717e-16, 7.9930e-16,\n 4.2238e-17, 4.3557e-15, 1.7987e-16, 8.0416e-15, 5.0414e-17, 2.9089e-17,\n 2.7856e-15, 1.0362e-16, 2.7662e-16, 2.1348e-17, 3.5580e-17, 3.8833e-14,\n 1.1375e-16, 2.0793e-15, 2.2186e-15, 9.2579e-16, 5.8988e-16, 4.7928e-17,\n 9.1309e-15, 5.6620e-17, 6.7969e-16, 1.3899e-15, 1.1042e-15, 5.4168e-18,\n 3.5461e-17, 7.5870e-18, 6.1821e-17, 9.0003e-16, 1.0563e-15, 1.2951e-14,\n 4.8220e-15, 2.1652e-15, 3.5354e-17, 2.5852e-15, 1.3585e-16, 4.7836e-15,\n 8.5831e-16, 3.8242e-15, 9.6107e-16, 4.6711e-15, 9.5832e-17, 8.8523e-15,\n 9.7181e-16, 7.0205e-17, 8.3403e-15, 2.5450e-18, 9.1231e-17, 1.7531e-16,\n 2.8283e-18, 1.1740e-17, 1.7615e-17, 5.1179e-18, 1.6274e-16, 1.2161e-15,\n 1.4625e-16, 5.0154e-18, 3.5758e-16, 9.5730e-16, 2.1561e-17, 5.8574e-15,\n 1.1961e-14, 1.3720e-17, 5.0718e-17, 5.1324e-15, 8.4921e-18, 3.9062e-16,\n 4.4363e-17, 6.8457e-17, 2.9437e-14, 4.3067e-14, 1.2284e-16, 5.3014e-17,\n 6.1020e-16, 1.9003e-17, 2.9545e-16, 2.1234e-16, 2.4617e-15, 5.9364e-17,\n 8.9280e-17, 1.8350e-16, 3.8169e-15, 9.9933e-17, 1.9243e-15, 1.8897e-16,\n 7.9135e-17, 3.2294e-16, 7.9037e-15, 9.3284e-18, 4.2946e-15, 3.5291e-15,\n 2.2788e-16, 3.2072e-16, 6.8013e-17, 3.1090e-17, 4.8642e-17, 4.6019e-16,\n 2.3277e-16, 1.4173e-16, 3.3730e-16, 3.2942e-15, 1.9252e-15, 5.3745e-17,\n 2.8491e-15, 1.7991e-17, 7.4710e-17, 2.6422e-16, 1.3799e-16, 1.6780e-16,\n 2.4225e-17, 4.3385e-17, 1.2904e-15, 9.1536e-17, 1.1376e-15, 2.1557e-16,\n 2.5046e-14, 1.8427e-15, 4.4081e-17, 3.9532e-15, 2.1188e-15, 2.0025e-16,\n 9.6670e-17, 3.9217e-16, 1.0157e-15, 4.0273e-17, 3.1556e-17, 5.4627e-16,\n 1.0418e-15, 2.5445e-16, 5.7482e-17, 5.5744e-16, 2.7719e-17, 8.7998e-15,\n 3.6143e-18, 2.5635e-15, 1.5279e-15, 6.7636e-17, 2.5562e-16, 3.9025e-15,\n 7.8504e-15, 7.7198e-17, 1.1770e-14, 2.0357e-16, 1.5098e-14, 7.0576e-16,\n 4.2207e-17, 2.0444e-15, 1.9508e-15, 1.7749e-15, 9.9684e-16, 4.6252e-16,\n 1.5896e-16, 1.0984e-14, 3.8324e-15, 5.0621e-15, 1.4738e-16, 5.2971e-16,\n 3.0243e-17, 1.1106e-15, 2.2910e-15, 8.5427e-17, 1.8188e-17, 3.9201e-17,\n 1.7412e-14, 2.4071e-17, 2.5160e-17, 5.7623e-16, 3.5854e-16, 1.2712e-15,\n 9.5295e-18, 3.2981e-15, 5.1218e-16, 1.2877e-16, 1.4970e-15, 4.2242e-16,\n 1.7951e-17, 5.3421e-17, 4.6013e-18, 1.2379e-16, 5.3726e-15, 3.9282e-17,\n 6.8545e-16, 2.8412e-16, 4.3858e-17, 5.7732e-18, 6.3547e-16, 2.7738e-15,\n 3.5464e-16, 2.0190e-17, 3.6897e-16, 4.8508e-18, 4.8759e-16, 6.5557e-16,\n 8.0666e-18, 7.1141e-16, 4.8020e-17, 6.4560e-17, 2.3798e-16, 3.3871e-15,\n 1.4315e-17, 1.0851e-15, 5.8474e-15, 5.6215e-17, 1.6733e-17, 3.1091e-16,\n 9.8393e-17, 2.4169e-15, 3.0677e-16, 1.4039e-16, 7.9054e-15, 7.2526e-17,\n 1.3738e-14, 2.1976e-17, 4.3030e-16, 2.5232e-16, 5.2069e-15, 5.3629e-15,\n 3.8830e-16, 1.8846e-15, 1.1204e-17, 9.4336e-16, 1.4616e-16, 4.0975e-17,\n 1.4230e-15, 1.4240e-16, 3.4795e-16, 6.3527e-15, 8.5784e-17, 1.1841e-16,\n 1.7105e-16, 4.9805e-15, 1.9730e-16, 2.9724e-17, 1.0275e-14, 1.3669e-15,\n 3.9841e-15, 2.0752e-17, 7.6193e-16, 2.8387e-16, 5.5473e-15, 4.9503e-16,\n 1.8313e-17, 1.1600e-15, 5.6577e-16, 3.8039e-17, 2.9850e-15, 1.1470e-14,\n 2.6619e-15, 6.7030e-15, 6.1613e-15, 4.8784e-16, 6.4030e-16, 5.3661e-17,\n 6.2059e-17, 9.3968e-18, 5.0431e-17, 7.2473e-17, 4.4814e-17, 1.6302e-17,\n 5.9682e-17, 1.0111e-14, 4.8269e-16, 6.3197e-16, 2.4372e-16, 9.9120e-16,\n 2.0190e-15, 8.6235e-15, 4.7071e-16, 9.3635e-16, 5.1110e-16, 3.8676e-15,\n 2.4428e-14, 2.9877e-16, 2.6188e-16, 5.7474e-17, 6.8423e-15, 4.8316e-14,\n 5.6202e-17, 2.4803e-17, 5.6178e-15, 2.8487e-17, 9.7873e-15, 1.6059e-17,\n 1.6695e-15, 7.2401e-17, 8.9717e-18, 2.0052e-17, 3.9647e-16, 5.1788e-15,\n 1.6357e-14, 1.2592e-15, 2.3288e-17, 3.3438e-16, 1.6593e-17, 6.7821e-16,\n 9.9705e-16, 1.0211e-17, 1.5867e-17, 3.5228e-16, 2.5889e-17, 7.3007e-16,\n 8.8292e-15, 4.2733e-16, 5.1612e-16, 4.2112e-17, 7.1184e-15, 1.4428e-15,\n 7.4478e-15, 3.0173e-16, 3.8750e-15, 6.5768e-15, 3.5661e-17, 1.2936e-14,\n 4.3886e-17, 5.7081e-15, 1.0550e-16, 1.0783e-16, 9.9581e-18, 1.8824e-14,\n 1.6724e-16, 9.4972e-15, 8.3082e-17, 4.0456e-15, 1.1280e-14, 1.6411e-16,\n 2.4384e-16, 6.9556e-16, 1.5904e-15, 8.8571e-16, 3.1766e-16, 3.6049e-17,\n 9.4075e-17, 1.0048e-15, 1.4254e-16, 3.6543e-16, 6.5894e-15, 6.7881e-16,\n 5.9337e-17, 3.3834e-15, 1.6332e-15, 1.3916e-17, 1.4453e-16, 6.3927e-17,\n 8.7753e-15, 4.8261e-15, 3.3028e-14, 5.8300e-18, 8.1144e-16, 5.8468e-18,\n 1.7585e-17, 1.2367e-14, 2.7003e-16, 1.8159e-18, 1.0068e-16, 8.1940e-17,\n 1.4431e-15, 1.2122e-15, 1.6770e-14, 2.6946e-16, 2.2381e-16, 6.7270e-17,\n 3.7892e-17, 1.0821e-16, 3.1479e-15, 3.0219e-16, 1.8400e-17, 3.3876e-15,\n 2.6507e-15, 1.1927e-16, 3.2443e-16, 1.1893e-17, 3.0174e-16, 2.0309e-16,\n 4.9162e-15, 1.0860e-17, 1.0026e-16, 2.2691e-17, 1.0156e-15, 2.1401e-16,\n 4.2727e-15, 9.7645e-17, 2.2465e-18, 1.2863e-17, 1.1837e-15, 1.1554e-15,\n 5.5322e-16, 2.4270e-17, 4.8629e-18, 2.1188e-17, 8.8659e-15, 3.4736e-18,\n 5.3979e-15, 1.8244e-15, 6.2138e-18, 3.6992e-16, 3.2418e-17, 4.7554e-17,\n 7.3232e-17, 5.2734e-18, 3.7255e-17, 1.3715e-17, 2.6004e-15, 2.4311e-17,\n 1.0101e-15, 2.1846e-14, 1.0369e-17, 2.7679e-16, 1.4273e-16, 9.2071e-15,\n 6.3733e-17, 1.7709e-16, 7.8156e-17, 7.6979e-15, 6.8636e-17, 5.6035e-16,\n 4.6028e-15, 1.2296e-16, 5.5318e-17, 3.4961e-16, 1.2801e-16, 1.9237e-16,\n 2.6973e-15, 2.7762e-17, 4.5215e-15, 1.0735e-14], device='cuda:0')" + }, + "61": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 1.8136e-13, 1.9867e-13, 1.2485e-13, 7.4530e-14, 1.1793e-13,\n 9.9256e-15, 2.9271e-15, 2.9500e-13, 1.2036e-13, 3.0567e-15,\n 5.0383e-14, -6.3370e-14, -6.0411e-16, 1.0744e-13, 1.4048e-13,\n 2.6467e-13, -2.1901e-14, 3.9577e-14, 2.4481e-13, -1.0066e-15,\n 6.3982e-14, -1.5437e-14, 4.1537e-14, 9.1887e-17, 1.2637e-13,\n 5.5467e-15, 3.8070e-14, 6.5776e-16, 1.2974e-13, 1.6544e-13,\n 7.5462e-14, 1.8820e-13, 7.4208e-15, 6.3005e-14, 1.7257e-13,\n -3.4283e-14, 1.0500e-13, 1.1484e-13, 1.6207e-13, 3.0328e-13,\n 1.9414e-13, 6.8684e-14, 3.0309e-14, -1.8474e-14, 1.1819e-13,\n 8.0478e-14, 1.1867e-13, 2.1103e-13, -7.5807e-15, 8.5640e-15,\n 1.6851e-13, -2.7883e-15, 6.2805e-14, 1.6902e-13, -2.2457e-14,\n -1.6982e-14, 1.9825e-13, 1.3877e-13, 1.2418e-13, 2.5109e-13,\n -3.0238e-14, 8.5045e-14, -1.1598e-15, -5.0283e-15, 1.5025e-13,\n 5.4460e-14, -2.7246e-14, -2.4042e-14, 2.8504e-14, 9.0527e-14,\n 2.3538e-13, 2.1592e-13, 4.0645e-14, 1.3741e-13, 1.4862e-13,\n -1.9285e-14, 1.9521e-13, 9.8259e-14, -9.7833e-16, 8.9458e-14,\n 3.3671e-14, 1.8896e-13, 1.6537e-13, -4.4169e-15, -1.3068e-14,\n 1.8536e-13, 2.3895e-13, 4.7868e-15, 1.2085e-15, 6.9205e-14,\n 5.8866e-14, 2.3499e-14, -1.3315e-13, -6.5161e-14, -1.2971e-14,\n 2.0607e-13, 1.5073e-14, 5.8161e-15, 1.1498e-13, -9.9409e-15,\n -1.4702e-14, 5.7557e-15, 3.9252e-14, 7.8164e-14, 1.1008e-14,\n 1.6957e-15, 1.0482e-13, -1.2778e-14, 2.5453e-13, -2.8546e-15,\n 8.8824e-16, -2.2675e-15, 6.1817e-14, 8.5543e-14, 7.4161e-14,\n -4.7160e-14, -5.5953e-15, 9.1385e-14, 2.7119e-14, 1.0056e-13,\n 2.1501e-13, 3.6337e-14, 6.0386e-14, 1.5061e-13, 8.3990e-14,\n 1.2567e-14, 5.8445e-14, 4.3127e-15, 7.5628e-14, 1.5164e-15,\n 3.1675e-14, 9.1584e-15, 1.0737e-14, 1.0803e-13, 1.0038e-13,\n -1.4131e-14, 1.3561e-13, -4.5291e-16, 2.2476e-13, -1.3582e-14,\n 2.3414e-13, -5.0827e-15, 2.7631e-15, 4.7229e-14, 1.3370e-14,\n -2.2403e-14, 7.4100e-14, 8.9453e-14, 2.5029e-13, 1.2029e-15,\n 9.3063e-14, -3.1887e-15, 3.2135e-13, -2.3121e-14, 3.3091e-13,\n 1.1999e-14, 1.8522e-13, -7.0335e-15, 3.4407e-14, -1.0339e-14,\n 1.2951e-13, 1.9276e-13, 4.1472e-14, 2.3889e-13, 3.4891e-14,\n -1.6401e-14, 1.0384e-13, -1.0104e-14, 6.0801e-14, 1.7407e-13,\n 2.0478e-13, 7.9438e-14, -8.3653e-14, 6.0588e-14, 2.5098e-13,\n -3.2461e-15, -3.7088e-14, 5.1533e-15, 2.8500e-14, 3.2894e-14,\n 1.5576e-13, 3.2297e-13, 1.5865e-13, 5.9509e-14, 1.1601e-13,\n 4.6721e-14, 3.4117e-14, -3.1554e-14, -4.5302e-15, -3.0458e-14,\n 5.3845e-14, 7.8244e-14, 1.0153e-13, -2.6366e-14, 8.5287e-15,\n 3.2936e-15, 1.2811e-13, 1.0836e-13, -7.2325e-16, 8.3024e-14,\n 1.4309e-13, 5.5366e-14, 8.9508e-14, -2.0240e-14, 7.0202e-14,\n -2.4230e-13, 4.7209e-14, -4.9467e-15, 9.8224e-15, 2.0758e-15,\n 1.1388e-13, 1.4389e-14, 4.8175e-14, 2.4801e-14, 1.5387e-13,\n 4.9642e-14, 1.2103e-13, 1.1160e-13, 1.2695e-13, 1.4510e-13,\n 3.6188e-14, 4.6393e-14, 6.2440e-14, 7.4658e-15, 2.2398e-14,\n -1.6787e-15, 5.8815e-14, 4.7574e-14, -3.9210e-14, 2.1202e-13,\n 1.5368e-13, 9.5463e-14, 4.0868e-14, 2.2449e-13, 4.1286e-14,\n -2.1769e-15, 2.0191e-14, -2.8883e-14, 1.2157e-13, 2.8645e-14,\n -2.4577e-14, 2.5037e-14, 4.1472e-15, 6.9337e-14, 1.6906e-13,\n 2.1159e-13, 1.3897e-13, 1.2460e-13, 2.1416e-13, 1.5699e-13,\n 9.2568e-15, -3.6152e-14, 2.0941e-13, 1.4208e-13, 8.1548e-14,\n 1.4577e-13, 2.6934e-14, 4.6102e-14, 5.6764e-14, 7.2812e-17,\n 3.6710e-14, 9.1664e-14, -1.8711e-15, 1.8149e-14, -2.0759e-14,\n -2.3998e-14, 7.7995e-15, 6.5776e-14, 2.2693e-15, 8.1544e-14,\n 4.0542e-14, 3.6096e-14, -9.5197e-15, 8.6896e-14, 2.1297e-14,\n 1.0193e-13, 1.6315e-14, -1.1926e-14, -2.5651e-14, 1.2352e-13,\n 2.3445e-15, -1.6143e-14, 9.6742e-15, -2.0588e-16, 1.8068e-13,\n 2.3731e-13, -9.8582e-15, 2.2968e-15, -1.3315e-13, 1.8302e-13,\n -2.5342e-14, 7.6475e-15, 6.3236e-14, -1.5795e-14, 1.1613e-15,\n -6.5067e-14, 5.6413e-14, 5.4517e-15, -3.7221e-14, 4.6745e-14,\n 4.5640e-15, 3.7374e-14, -1.9267e-14, -1.8979e-14, 1.1993e-15,\n 2.1916e-13, -2.9639e-15, 1.8539e-14, -3.2609e-14, 2.8332e-13,\n 1.0871e-16, 5.6386e-14, 6.0451e-15, 5.7729e-15, 9.1816e-14,\n 1.4222e-13, 1.8942e-14, 7.9765e-14, -9.5002e-14, -4.6553e-14,\n 1.8182e-13, 1.4497e-13, 3.2814e-14, -8.4954e-15, 2.3294e-13,\n 1.0181e-13, 3.4613e-13, 1.6857e-13, 8.4816e-14, -5.3984e-14,\n 2.3299e-13, 2.2616e-13, -9.4722e-15, 7.7425e-14, 2.1779e-14,\n 5.3907e-15, -2.5548e-14, 2.1282e-14, 3.5085e-13, -1.1811e-14,\n 5.5390e-14, -3.6518e-14, 1.0141e-13, 5.2164e-14, -2.5251e-15,\n -2.0174e-14, 1.2301e-13, 1.8537e-13, -5.5458e-15, 5.3059e-14,\n 6.9314e-14, -1.6656e-14, 1.0870e-13, 4.8067e-14, 2.1816e-13,\n -2.0927e-14, 5.1758e-14, -2.8675e-14, 1.5643e-13, 1.6420e-13,\n -4.1689e-15, 1.6424e-13, 3.4152e-14, -1.2851e-13, 3.9643e-13,\n 2.2754e-13, 2.7185e-13, 1.2697e-13, 5.0129e-14, 1.2446e-13,\n 2.2411e-14, 7.8177e-14, 5.3791e-14, 2.1601e-13, 4.5718e-14,\n 2.4433e-14, 2.3262e-14, 4.1025e-13, 1.3296e-13, -3.0460e-16,\n 4.9615e-14, 1.7869e-14, 1.3142e-15, 3.4142e-14, 8.3919e-15,\n -5.8534e-15, 6.6281e-14, 3.5739e-15, 8.1761e-14, 7.9953e-14,\n 8.5041e-14, -1.4965e-15, 5.5840e-15, 9.3482e-15, 7.8139e-14,\n 1.1843e-13, 2.2788e-13, 2.0904e-16, 3.7212e-14, 1.5303e-13,\n 1.0757e-13, 4.9094e-14, 4.1195e-14, 6.3742e-14, -4.0881e-14,\n -7.7134e-15, 2.3654e-14, 2.1593e-14, 1.3646e-14, 2.4640e-13,\n 5.4312e-14, -3.7280e-15, 4.8649e-14, -1.6913e-14, -1.0911e-16,\n 9.0469e-14, 3.5019e-13, 6.8223e-14, 7.0482e-14, 1.0433e-13,\n 1.6760e-15, 2.0077e-13, 2.1365e-13, 2.7820e-13, 3.0947e-13,\n 1.0052e-13, 1.8743e-13, 6.6801e-14, -8.1242e-15, 1.2358e-14,\n 7.8545e-14, 1.7837e-13, 1.2706e-13, 1.1393e-13, -1.0017e-14,\n 1.5859e-13, 1.6347e-14, 3.2101e-13, 1.1559e-13, -4.0467e-15,\n 5.8939e-14, -1.0738e-14, 9.3142e-15, 2.1051e-15, -3.5074e-14,\n 4.1616e-14, 6.2014e-15, -2.6838e-14, 2.3754e-13, 1.5115e-13,\n 5.1873e-14, 6.6816e-14, 9.8495e-14, -5.0742e-15, 5.3990e-14,\n 9.7862e-15, 5.9319e-14, 1.2030e-13, -2.3237e-14, 4.2522e-15,\n 1.4536e-13, 1.4245e-13, 1.1483e-14, 1.1858e-13, 9.8976e-14,\n -3.1741e-14, -2.7206e-14, 4.2030e-15, -2.8227e-14, 3.6009e-15,\n 5.5953e-15, 6.2904e-14, 2.9434e-14, 1.4284e-14, -2.7055e-15,\n 2.3628e-14, -4.7146e-14, 2.1207e-14, 6.1033e-14, 8.7700e-15,\n 8.8524e-14, 4.2847e-15, 3.3050e-13, 3.9000e-15, 3.3953e-13,\n 3.2540e-14, 3.3013e-13, 3.7609e-14, 6.7891e-14, 1.1297e-14,\n -5.1811e-14, 3.6553e-13, 2.8999e-14, 7.8277e-14, 4.8579e-14,\n -8.3141e-15, -2.7850e-14, 2.9828e-14, -7.7477e-14, -9.3578e-16,\n 1.1769e-14, 4.0604e-14, 1.8971e-13, -7.4497e-15, 1.2198e-13,\n 7.2338e-14, 3.0302e-15, 3.4286e-14, 1.7869e-14, 1.1293e-13,\n 1.4565e-16, 1.1176e-13, 1.0894e-14, -1.0539e-13, -1.0374e-15,\n 4.0586e-14, 1.2745e-13, -6.8788e-14, 1.0904e-13, -4.1939e-15,\n 1.7897e-13, 6.5488e-14, 6.1417e-14, 1.1749e-13, 2.6011e-14,\n 9.4217e-14, -1.0681e-14, 1.4732e-13, -2.7541e-15, 4.2625e-14,\n 3.1119e-14, 1.4514e-13, -2.3859e-15, 3.0348e-13, 1.1106e-13,\n 1.7342e-13, 1.2250e-13, 2.1475e-13, 3.9222e-14, 1.1519e-13,\n 4.4951e-14, 1.4947e-13, 3.8927e-14, 9.1539e-14, 2.2548e-14,\n -1.3185e-15, 1.0334e-13, 7.2450e-14, 1.8041e-13, -1.3689e-15,\n 1.2971e-13, 1.5297e-13, 5.7931e-14, 4.3276e-14, -1.3753e-14,\n -1.0019e-14, 1.7552e-14, 6.1959e-14, 5.2401e-14, 1.8815e-13,\n 4.5683e-13, 7.8200e-16, -5.9898e-15, 2.6778e-14, 6.4379e-15,\n 6.5977e-14, 1.4147e-13, 5.3407e-14, -4.0008e-15, -2.5470e-14,\n 2.6092e-13, 1.1323e-13, 2.1015e-13, 1.2982e-13, -3.4465e-14,\n 9.9443e-14, -1.5960e-14, -1.1212e-14, 2.2951e-13, 1.1338e-13,\n 1.8938e-14, 4.0337e-14, 2.2424e-13, 8.0338e-14, 1.8833e-13,\n 1.8633e-14, 2.7920e-14, 1.4172e-14, -2.6220e-15, -1.6992e-14,\n -6.3910e-15, 2.3566e-13, 1.1825e-13, 4.0289e-14, 3.7389e-14,\n 2.7357e-14, 2.9717e-14, -3.6376e-14, 1.9280e-13, 5.1407e-14,\n 1.4959e-14, 8.7294e-14, 5.1205e-14, 9.9775e-15, -2.4982e-14,\n 8.4965e-15, -3.2532e-15, 2.0736e-13, 3.3653e-15, 1.0841e-13,\n -1.1076e-15, 2.3718e-15, 2.6023e-14, 4.7905e-14, 6.9202e-15,\n -1.1163e-14, 1.5431e-13, 5.1396e-15, 3.7742e-14, 3.1811e-13,\n -1.0951e-14, -7.5561e-15, 2.1914e-13, 4.4253e-13, 2.6579e-14,\n -6.1468e-15, -3.6165e-14, 5.1667e-14, 1.5149e-13, 3.9978e-15,\n 2.5623e-13, 1.5803e-14, 3.0697e-15, 3.5039e-13, 4.1878e-14,\n 1.2363e-13, 9.7450e-15, 6.1397e-15, -1.3359e-13, 1.7555e-13],\n device='cuda:0')", + "exp_avg_sq": "tensor([4.3211e-15, 4.2666e-15, 7.3906e-15, 8.1749e-16, 6.1560e-16, 2.4203e-15,\n 4.6288e-17, 1.4699e-15, 1.0415e-14, 2.2711e-16, 3.9608e-15, 1.4252e-15,\n 3.7153e-18, 2.7806e-15, 7.4803e-16, 1.0157e-15, 2.6831e-17, 3.9568e-15,\n 1.6464e-15, 7.6383e-17, 1.3962e-15, 2.6760e-17, 4.0336e-15, 3.9343e-15,\n 1.6644e-15, 2.9952e-16, 5.6635e-15, 1.7581e-18, 3.3362e-15, 1.6349e-15,\n 8.4417e-17, 6.0540e-16, 3.0965e-15, 1.3685e-15, 3.3675e-15, 1.6638e-15,\n 3.2893e-15, 1.9752e-16, 9.1912e-15, 1.0288e-14, 1.0082e-15, 2.0299e-16,\n 9.2292e-16, 2.6022e-17, 7.6999e-18, 1.8363e-15, 8.5795e-15, 1.9637e-15,\n 2.5600e-17, 7.5606e-16, 3.4886e-16, 3.5688e-17, 1.7766e-15, 3.5127e-17,\n 2.0301e-16, 4.2413e-17, 1.0741e-16, 8.7818e-16, 3.7892e-15, 3.3592e-16,\n 4.6461e-15, 2.4146e-15, 5.3018e-17, 2.9961e-16, 5.5004e-16, 3.1625e-17,\n 2.9052e-15, 3.4327e-15, 3.0568e-15, 2.8813e-15, 9.6102e-16, 2.8973e-15,\n 2.4104e-15, 2.2221e-15, 7.7687e-15, 9.4785e-17, 9.0093e-16, 1.0406e-15,\n 3.3874e-18, 6.6713e-15, 2.8174e-15, 1.5796e-16, 4.7967e-16, 3.4116e-17,\n 2.5647e-16, 2.5734e-15, 2.9497e-15, 5.7351e-15, 5.4200e-18, 1.5039e-15,\n 7.2939e-16, 3.1270e-15, 3.0024e-15, 9.7802e-16, 6.2935e-16, 3.0561e-16,\n 6.4351e-16, 2.6677e-15, 1.5725e-15, 1.5078e-17, 1.0662e-15, 2.4525e-16,\n 2.8183e-15, 2.0349e-15, 2.6741e-16, 7.6725e-16, 1.8048e-14, 2.6158e-17,\n 7.3119e-16, 1.6146e-17, 1.1483e-15, 2.9826e-16, 4.9075e-17, 2.9866e-15,\n 1.2573e-15, 2.9631e-17, 2.7089e-17, 4.0485e-16, 8.0826e-16, 2.9054e-16,\n 4.9746e-15, 3.9520e-15, 4.6493e-16, 2.0347e-16, 1.9055e-15, 3.8393e-15,\n 5.0682e-16, 2.5242e-17, 1.3761e-16, 2.4481e-15, 3.8400e-16, 1.4470e-15,\n 4.4424e-15, 2.2577e-15, 1.0425e-15, 2.2947e-16, 2.4476e-15, 7.3666e-16,\n 1.1728e-15, 8.5709e-17, 2.2301e-15, 3.1717e-16, 5.7684e-17, 1.8537e-15,\n 8.6950e-15, 4.5079e-17, 6.5130e-16, 3.2870e-15, 1.1367e-15, 1.1902e-16,\n 2.1338e-15, 1.3079e-17, 2.2905e-16, 1.9224e-17, 3.1521e-15, 8.0095e-16,\n 2.1139e-15, 3.3797e-16, 2.3800e-15, 2.9202e-18, 2.3343e-15, 2.2744e-15,\n 8.1022e-16, 6.1094e-17, 6.6199e-15, 2.1249e-15, 1.1105e-15, 1.2832e-15,\n 2.2033e-15, 4.5376e-15, 5.8012e-15, 2.3743e-15, 1.4543e-15, 2.2610e-15,\n 4.0657e-15, 3.1028e-16, 1.1575e-16, 1.8078e-15, 6.7544e-15, 6.9392e-16,\n 9.9990e-16, 1.8724e-15, 1.4763e-15, 4.6302e-15, 9.8537e-15, 3.3374e-15,\n 2.1923e-16, 1.2799e-15, 7.6512e-16, 4.1952e-15, 2.6257e-15, 2.2590e-16,\n 1.1987e-16, 1.1597e-15, 3.7690e-15, 2.2246e-15, 2.9008e-15, 3.7610e-15,\n 4.7792e-15, 2.9033e-15, 4.3042e-16, 3.9006e-15, 1.6984e-16, 4.7714e-16,\n 3.8676e-16, 3.9993e-15, 2.0023e-15, 2.2240e-16, 4.0913e-16, 4.6105e-16,\n 1.6116e-15, 4.6705e-17, 1.3006e-15, 2.1974e-15, 1.4444e-15, 1.7347e-15,\n 5.8533e-16, 3.8794e-15, 6.8498e-16, 9.9568e-15, 5.1237e-16, 2.4308e-17,\n 5.8756e-15, 2.6165e-15, 6.5042e-16, 1.2935e-16, 8.3336e-16, 4.3618e-15,\n 4.4757e-17, 5.3839e-15, 2.1445e-15, 1.8994e-15, 1.4865e-15, 2.6952e-17,\n 4.0929e-15, 2.3261e-17, 1.3631e-15, 1.6883e-15, 1.8250e-15, 2.7235e-16,\n 2.0615e-17, 1.8054e-16, 1.4703e-15, 5.9093e-16, 1.2385e-15, 2.2409e-15,\n 7.2345e-15, 4.5225e-15, 4.4342e-16, 4.2890e-15, 5.9038e-16, 2.8494e-15,\n 1.2528e-15, 6.8081e-15, 1.5629e-15, 5.5985e-15, 4.2494e-17, 7.4568e-15,\n 1.8681e-15, 2.3432e-17, 2.8838e-15, 1.3713e-16, 2.3635e-17, 7.2000e-17,\n 2.9188e-17, 5.5710e-16, 2.1773e-17, 5.1291e-16, 9.6113e-16, 1.3297e-15,\n 7.6503e-16, 4.7581e-17, 1.5314e-15, 2.7519e-15, 4.8092e-16, 5.8715e-15,\n 3.5275e-15, 3.3340e-16, 2.3479e-16, 6.2924e-15, 1.0724e-17, 7.6763e-16,\n 4.0499e-16, 9.1771e-16, 2.2593e-14, 1.1275e-14, 1.0344e-15, 1.2424e-17,\n 1.8920e-15, 2.3974e-16, 2.9273e-15, 1.5774e-15, 1.5821e-15, 4.2156e-17,\n 3.0912e-17, 9.3173e-16, 2.4930e-15, 5.4091e-17, 3.2060e-15, 8.9530e-16,\n 4.5203e-16, 1.1874e-15, 7.0011e-15, 2.2312e-16, 2.3505e-15, 1.7782e-15,\n 5.8018e-17, 1.1112e-16, 2.5062e-17, 2.9671e-16, 1.3042e-17, 9.9411e-16,\n 8.3931e-16, 1.9781e-15, 1.8355e-15, 4.2723e-15, 3.1278e-15, 1.7428e-17,\n 1.9004e-15, 8.1005e-16, 4.8517e-16, 6.3662e-16, 7.3661e-16, 6.6487e-17,\n 3.2409e-16, 5.6235e-16, 1.8324e-15, 4.2166e-17, 3.7397e-15, 1.1724e-15,\n 6.4520e-15, 3.2321e-15, 6.7533e-16, 2.7827e-15, 5.3710e-15, 7.8843e-16,\n 1.6615e-15, 1.7548e-15, 3.2784e-15, 1.4026e-17, 1.1669e-17, 6.3479e-16,\n 3.7380e-15, 4.0747e-16, 1.6919e-17, 1.6619e-15, 9.7197e-17, 6.2989e-15,\n 1.1103e-16, 3.7931e-15, 1.8750e-15, 3.0751e-17, 1.2260e-15, 8.1753e-15,\n 3.9946e-15, 4.3080e-17, 4.3190e-15, 1.5107e-15, 6.2814e-15, 7.0142e-16,\n 9.9607e-18, 2.8592e-15, 2.2956e-15, 2.6836e-15, 1.6111e-15, 1.4191e-15,\n 7.0312e-16, 6.9324e-15, 5.4607e-15, 4.5813e-15, 6.7139e-16, 1.6496e-15,\n 9.9789e-16, 2.5187e-15, 2.1422e-15, 1.0459e-15, 2.1981e-16, 2.6192e-17,\n 1.1757e-14, 1.7368e-17, 2.8591e-17, 1.7225e-15, 9.8444e-16, 3.7248e-15,\n 2.1719e-16, 3.5924e-15, 1.3617e-15, 1.0660e-15, 2.1689e-15, 1.1538e-15,\n 3.9448e-17, 1.3684e-15, 2.9235e-16, 5.8025e-16, 3.9485e-15, 3.3688e-17,\n 1.7333e-15, 9.3624e-17, 2.7603e-17, 1.6151e-16, 2.3100e-15, 3.3632e-15,\n 1.1337e-15, 2.5041e-16, 2.5143e-15, 9.1547e-17, 2.2368e-15, 1.7276e-15,\n 2.9411e-16, 2.3093e-15, 1.9056e-16, 1.0571e-15, 7.9935e-16, 7.3852e-15,\n 3.2638e-18, 1.5974e-15, 4.8400e-15, 1.1419e-15, 8.8020e-16, 7.2320e-16,\n 4.0616e-17, 4.0090e-15, 1.5318e-15, 8.3213e-16, 3.5002e-15, 2.6750e-16,\n 5.8565e-15, 4.7455e-16, 1.1323e-15, 7.1619e-17, 3.4811e-15, 6.8486e-15,\n 1.6637e-15, 4.8136e-15, 2.6350e-18, 3.0050e-15, 6.2905e-17, 4.6923e-16,\n 9.6099e-16, 6.6602e-17, 1.0604e-15, 2.1819e-15, 6.2508e-16, 7.8699e-16,\n 6.3522e-17, 1.6596e-15, 4.6388e-17, 9.8224e-16, 3.6774e-15, 5.4610e-15,\n 2.2139e-15, 1.0401e-17, 3.1841e-15, 1.4217e-15, 4.4556e-15, 1.2204e-15,\n 3.6417e-16, 2.3329e-15, 2.3658e-15, 6.2000e-16, 6.6548e-15, 3.9046e-15,\n 4.0754e-15, 2.4917e-15, 5.1954e-15, 3.9771e-15, 1.2943e-15, 9.9756e-17,\n 3.8984e-17, 3.8392e-16, 1.1092e-17, 3.3528e-16, 6.7119e-16, 2.8714e-16,\n 2.1700e-16, 2.6546e-15, 1.5807e-15, 2.7261e-15, 1.0988e-15, 1.8387e-15,\n 1.7405e-15, 6.9324e-15, 8.5483e-16, 2.5339e-15, 1.4471e-15, 1.8681e-15,\n 1.1077e-14, 8.1193e-16, 5.5226e-16, 9.3784e-16, 8.7696e-15, 2.4208e-14,\n 3.4370e-16, 1.2240e-16, 6.2409e-15, 9.5211e-18, 6.4183e-15, 1.8020e-16,\n 1.9509e-15, 3.8853e-17, 1.0227e-16, 1.7682e-16, 1.1856e-15, 6.1832e-15,\n 4.7130e-15, 2.7710e-15, 4.9696e-16, 1.4075e-15, 2.2672e-16, 1.2612e-15,\n 4.0045e-15, 2.1439e-16, 8.3783e-16, 2.9025e-15, 4.5888e-18, 3.1387e-15,\n 4.1011e-15, 2.1437e-15, 1.2600e-15, 1.2169e-17, 4.7189e-15, 9.0587e-16,\n 5.8757e-15, 9.6339e-16, 3.0600e-15, 4.9251e-15, 1.5628e-17, 6.2782e-15,\n 1.4403e-17, 7.7977e-15, 3.9890e-16, 7.0418e-16, 3.8210e-16, 2.2213e-14,\n 2.3579e-15, 3.7996e-15, 6.3654e-16, 5.0970e-15, 5.0179e-15, 5.5859e-16,\n 1.9809e-15, 1.1717e-15, 1.3872e-15, 1.2010e-15, 1.0409e-15, 7.7891e-18,\n 7.4770e-16, 1.3093e-15, 1.2210e-16, 1.4851e-15, 3.3286e-15, 2.0019e-15,\n 5.2318e-16, 3.3691e-15, 1.9269e-15, 4.6778e-18, 3.3190e-16, 1.1497e-15,\n 4.0237e-15, 4.8829e-15, 1.7178e-14, 1.8057e-16, 2.5897e-15, 9.5250e-16,\n 5.0734e-16, 3.3631e-15, 1.3077e-15, 2.1527e-17, 2.7896e-17, 2.8662e-17,\n 8.6659e-16, 3.0644e-15, 2.0713e-14, 1.0262e-15, 1.1985e-15, 8.8436e-16,\n 4.9451e-16, 1.2916e-15, 4.1310e-15, 1.0422e-15, 2.5482e-16, 2.2052e-15,\n 3.9579e-15, 8.6356e-16, 1.3854e-15, 1.3148e-16, 8.4294e-16, 1.8299e-15,\n 5.0597e-15, 1.8505e-16, 2.4558e-17, 2.2268e-16, 2.9807e-15, 6.7549e-16,\n 3.9943e-15, 1.0472e-15, 1.3686e-17, 1.2860e-16, 2.7822e-15, 1.9133e-15,\n 6.8642e-16, 1.1599e-16, 6.8605e-17, 3.0037e-16, 1.0573e-14, 1.6606e-16,\n 3.9450e-15, 2.7483e-15, 3.5167e-17, 1.9720e-15, 8.9004e-18, 9.1345e-18,\n 2.6791e-15, 2.7392e-16, 4.0385e-16, 2.1937e-16, 3.4933e-15, 3.4790e-16,\n 2.4862e-15, 7.3527e-15, 2.3834e-16, 1.5355e-15, 5.6329e-16, 5.8350e-15,\n 2.4626e-17, 1.1967e-15, 1.1516e-15, 9.3355e-15, 4.6897e-16, 3.4381e-15,\n 4.3639e-15, 5.0259e-16, 3.8749e-16, 1.0692e-15, 4.4777e-17, 2.0695e-15,\n 1.1199e-15, 2.7752e-17, 3.9997e-15, 9.3480e-15], device='cuda:0')" + }, + "62": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 2.1813e-14, 3.6252e-14, 7.3629e-15, ..., 4.0748e-15,\n 1.3858e-14, 1.6640e-15],\n [ 4.9297e-14, 6.5973e-14, 2.5430e-14, ..., 7.9262e-15,\n 1.7160e-14, 2.8107e-14],\n [-7.1447e-14, -1.2339e-13, -5.1610e-14, ..., -1.6893e-14,\n -5.4427e-14, -7.8389e-14],\n ...,\n [-7.7837e-14, -1.8956e-13, -1.1704e-13, ..., -7.3551e-14,\n -1.5304e-13, -2.7866e-13],\n [ 9.2026e-14, 2.5407e-13, 1.0823e-13, ..., 9.0442e-14,\n 2.1534e-13, 3.5654e-13],\n [-1.0844e-14, -3.6080e-14, -3.5718e-14, ..., -2.3531e-14,\n -3.0453e-14, -7.1417e-14]], device='cuda:0')", + "exp_avg_sq": "tensor([[2.0728e-16, 8.9716e-17, 1.7842e-16, ..., 4.9407e-17, 8.4720e-17,\n 1.5020e-16],\n [1.7105e-16, 5.0402e-17, 5.5218e-17, ..., 1.9028e-17, 8.9185e-17,\n 7.3881e-17],\n [4.0744e-16, 2.4401e-16, 2.3919e-16, ..., 1.8892e-16, 8.1547e-17,\n 1.3231e-17],\n ...,\n [2.8140e-13, 4.8817e-14, 1.6105e-13, ..., 1.0572e-14, 3.3332e-14,\n 1.0906e-14],\n [3.9949e-14, 6.5228e-15, 2.8891e-14, ..., 9.3977e-16, 1.0070e-15,\n 8.0532e-16],\n [2.6249e-13, 4.6552e-14, 1.3991e-13, ..., 2.3954e-14, 4.8850e-14,\n 2.4021e-14]], device='cuda:0')" + }, + "63": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 4.3969e-14, 7.5770e-14, -1.4903e-13, ..., -3.6988e-13,\n 4.6788e-13, -8.3189e-14], device='cuda:0')", + "exp_avg_sq": "tensor([4.4320e-16, 3.1053e-16, 5.8314e-16, ..., 5.2048e-13, 5.4378e-14,\n 7.1071e-13], device='cuda:0')" + }, + "64": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[ 1.4563e-13, -1.6049e-13, -4.0371e-13, ..., -5.2273e-14,\n -9.0607e-14, 2.3113e-14],\n [ 1.4161e-13, -1.1070e-13, -2.8791e-13, ..., -4.5977e-14,\n -6.5850e-14, 2.0878e-14],\n [ 1.8815e-13, -1.5442e-13, -3.8053e-13, ..., -6.1511e-14,\n -9.6447e-14, 2.7173e-14],\n ...,\n [-1.6643e-13, 1.5955e-13, 3.8231e-13, ..., 5.6992e-14,\n 8.1369e-14, -2.4583e-14],\n [ 2.2381e-13, -2.1464e-13, -4.9008e-13, ..., -6.8385e-14,\n -1.1312e-13, 2.9202e-14],\n [ 5.9059e-14, -1.0927e-14, -2.9557e-14, ..., -5.4357e-15,\n -6.2659e-15, 6.5935e-15]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.1601e-15, 4.4196e-14, 8.1827e-14, ..., 5.6222e-14, 5.7605e-14,\n 1.1483e-13],\n [7.3148e-15, 5.9487e-14, 6.2985e-14, ..., 5.0467e-14, 6.6134e-14,\n 2.3424e-14],\n [7.1338e-15, 1.8227e-14, 4.8223e-14, ..., 3.6568e-14, 3.5940e-14,\n 8.0017e-14],\n ...,\n [4.1146e-15, 2.2465e-14, 2.9639e-14, ..., 1.5057e-14, 3.4918e-14,\n 1.4214e-14],\n [1.3878e-14, 2.0124e-14, 8.0482e-14, ..., 6.2433e-14, 3.3626e-14,\n 2.2336e-13],\n [1.9545e-15, 2.4052e-15, 6.0642e-15, ..., 6.8935e-15, 1.8170e-14,\n 6.0056e-15]], device='cuda:0')" + }, + "65": { + "step": "tensor(8764.)", + "exp_avg": "tensor([-5.4729e-13, -4.5157e-13, -5.9266e-13, -6.2419e-13, -6.8289e-13,\n -5.1876e-13, -3.1651e-13, 9.9616e-13, 5.2877e-13, 1.2699e-13,\n -6.4401e-13, 2.3100e-13, -6.3097e-13, 1.1876e-13, -1.9782e-13,\n -5.6069e-13, -5.4736e-14, 1.7899e-13, 2.7095e-13, -5.4259e-13,\n -1.0583e-12, 3.9305e-13, -3.2351e-14, 6.2362e-13, 8.7733e-13,\n -3.2002e-13, -6.3209e-13, -3.4061e-13, -3.5698e-13, 9.4040e-13,\n -4.9824e-13, -6.5935e-13, -1.9464e-13, -1.6424e-13, -2.1757e-13,\n 5.6435e-13, -6.9732e-13, -1.2651e-13, 3.9692e-13, -8.0615e-14,\n 7.7912e-13, 1.7999e-13, -8.3926e-14, 8.8818e-13, 1.2134e-12,\n -4.2612e-13, -7.1607e-13, -2.4172e-13, -1.5415e-13, -1.1909e-14,\n -3.8260e-13, -3.0126e-13, -4.1308e-13, -4.1267e-13, 9.1844e-13,\n -1.5885e-13, 2.0054e-13, -2.9088e-13, 5.3233e-13, 6.5028e-13,\n 3.5251e-14, -1.8632e-13, -9.5185e-14, -8.9090e-14, 5.8195e-13,\n -2.2507e-13, 3.1784e-13, -3.6907e-13, -4.7665e-13, -5.6855e-13,\n 3.4067e-13, -8.7178e-13, -1.7446e-13, -7.5501e-13, -6.7701e-13,\n 4.2735e-13, 6.8147e-13, 8.2019e-14, 3.9255e-13, 5.1042e-14,\n 2.7524e-13, 6.7258e-13, 3.2306e-13, -7.5623e-13, -2.0136e-13,\n 1.8330e-13, 3.7365e-14, 1.7850e-13, -3.6012e-13, -5.2300e-13,\n -8.6654e-13, -6.7621e-13, -1.0333e-12, 7.3648e-13, -4.7457e-13,\n 6.7803e-14, -5.3121e-13, -3.9323e-13, 6.2648e-13, -3.5417e-13,\n -2.2391e-13, -3.8449e-13, 2.4309e-13, 4.7189e-14, -4.9425e-13,\n -5.0917e-13, 3.6957e-13, 4.1114e-13, 6.9442e-15, 8.6263e-13,\n -7.9071e-13, -3.6490e-13, 2.7120e-13, 8.6404e-14, 7.1664e-13,\n 7.3815e-13, -4.0890e-13, -2.6042e-13, -1.9017e-13, -3.4630e-13,\n -7.2154e-13, 1.3032e-13, 1.2999e-13, 6.8491e-13, 1.4276e-13,\n -2.0737e-13, 2.9792e-14, -4.4278e-13, 1.3340e-13, -4.7814e-13,\n -4.1728e-13, 2.1140e-13, -7.3223e-13, 8.6571e-13, -2.8506e-13,\n 6.3785e-13, -2.0967e-13, 5.1426e-13, -7.7220e-13, -5.4322e-13,\n -1.8718e-13, 1.0585e-12, 9.5690e-13, -4.4497e-13, 1.7815e-14,\n 9.1045e-14, 6.4101e-13, -4.4817e-13, -4.5299e-13, 4.5435e-13,\n 3.8081e-13, 9.6894e-13, 3.8242e-13, 6.1755e-13, -3.4767e-13,\n -6.7027e-13, 2.4184e-13, -5.7448e-13, 5.1804e-13, -3.3292e-13,\n 5.3268e-13, 2.7013e-13, 1.1188e-12, 2.9221e-13, -3.7024e-13,\n 7.1077e-13, 4.1613e-13, -8.1561e-13, 6.9401e-14, 1.5533e-13,\n -2.0252e-13, -6.9208e-14, -2.0803e-13, -2.3470e-13, -4.4346e-13,\n -5.6551e-13, 4.0409e-13, -1.4261e-13, 2.5053e-13, 3.3044e-13,\n -5.2879e-14, -4.5597e-13, -5.9541e-13, 8.4769e-13, 3.7159e-13,\n -2.5254e-13, -1.8060e-13, 8.0154e-14, -4.6469e-13, -3.0145e-13,\n -6.8002e-15, 2.3287e-13, 9.1956e-13, -2.9788e-13, 4.3368e-13,\n -1.2375e-13, 3.3694e-14, -7.8749e-15, -1.1187e-12, -5.0542e-13,\n 3.2464e-13, -1.1415e-13, -1.0597e-13, 6.5032e-13, -8.1838e-13,\n -7.5765e-13, 8.3367e-14, -5.3391e-13, 5.9026e-13, 4.9687e-13,\n -8.2188e-14, -3.3705e-13, 5.1564e-13, 5.2846e-13, 5.8269e-13,\n 2.3708e-13, 2.4326e-14, 2.7119e-13, 4.5241e-13, -3.1506e-14,\n -2.8837e-13, -5.7039e-13, -4.5772e-13, 1.3303e-13, -2.2092e-13,\n -3.3711e-13, 4.3721e-13, 2.0241e-13, -8.0497e-13, 1.1040e-13,\n 2.1626e-13, -4.6605e-13, 1.0777e-12, -2.2624e-13, -4.0614e-13,\n 5.5956e-14, -1.2535e-12, -5.9026e-13, -1.4015e-13, -7.2622e-13,\n 8.1575e-13, -4.4418e-13, -4.2855e-15, -6.0873e-13, 3.3647e-13,\n 7.8801e-13, -5.7710e-13, -3.3572e-13, 6.3561e-13, 2.3212e-13,\n -8.5607e-13, -1.2160e-12, 3.2734e-13, -8.1310e-13, -7.0638e-14,\n -6.5802e-13, -3.3917e-13, 1.2137e-13, 1.3066e-12, -6.0547e-13,\n -3.8904e-13, -2.9219e-13, -5.9565e-13, 1.4842e-13, 3.5919e-13,\n -9.6172e-14, -3.4710e-13, -1.1466e-12, -1.3745e-13, -7.7285e-13,\n 2.2032e-13, 4.8088e-13, -6.6308e-13, -2.7917e-13, -9.8281e-13,\n -1.8858e-13, -9.7806e-14, -8.6432e-13, 5.5731e-13, -9.0515e-13,\n -5.6431e-13, -5.7798e-13, 3.2393e-13, 1.1641e-12, 3.0551e-13,\n -1.3010e-13, -2.3967e-13, -4.8572e-13, -2.8439e-13, -1.6137e-13,\n -2.3904e-13, 8.1003e-13, 2.9102e-13, 1.1139e-12, -3.0477e-13,\n 3.1923e-13, 5.0324e-13, 4.4465e-13, -4.4171e-13, 1.1205e-13,\n 5.9813e-13, -1.0537e-12, 5.3095e-13, -3.2394e-13, 5.4940e-13,\n 2.7451e-13, 1.0591e-13, -9.2931e-14, 7.9310e-13, 6.4383e-14,\n 2.1853e-13, 1.2611e-12, -8.2849e-13, 6.1365e-13, -2.7718e-13,\n 1.1487e-12, -4.4832e-13, 1.0569e-13, -1.4205e-13, 2.9934e-13,\n -1.9793e-13, -9.9281e-13, 4.5313e-13, 7.5681e-13, -2.6473e-13,\n 8.7155e-13, -3.1928e-13, -5.0811e-13, 2.5258e-13, -2.6417e-13,\n -1.0475e-12, -7.0065e-13, -6.3843e-13, 3.2444e-13, -4.2525e-13,\n -4.3003e-13, -3.7018e-13, -2.2755e-13, -7.9052e-13, 5.2053e-13,\n 5.2905e-13, -3.9209e-13, 1.6872e-13, -3.8501e-13, -6.4508e-13,\n 1.7748e-13, -1.2821e-13, 3.6422e-13, -1.1209e-12, 8.8067e-14,\n 5.1516e-13, -1.7553e-13, 2.6359e-13, 6.5969e-13, -2.9568e-13,\n -7.4403e-14, -1.0771e-12, 2.1723e-14, -3.2405e-14, 7.2913e-14,\n -1.9576e-15, 4.6198e-13, 2.0877e-13, 4.6475e-14, -6.8707e-13,\n -7.1096e-13, -7.9101e-13, -1.3385e-13, 1.0102e-12, -3.2460e-13,\n 7.5237e-14, -3.0855e-13, -6.9297e-13, -1.6503e-13, 2.9596e-13,\n 1.0587e-12, 6.6002e-13, 3.6058e-13, -1.4822e-13, -2.6137e-13,\n 2.5141e-13, -1.0030e-13, 2.1871e-14, -1.0621e-12, -1.0523e-13,\n -9.0752e-13, 1.9397e-13, 2.5367e-13, 2.2215e-13, 9.6154e-13,\n -5.2043e-13, 8.1712e-13, -1.1393e-13, -2.3007e-13, -4.1073e-13,\n -2.7724e-13, 3.1846e-14, 2.2749e-13, -7.2115e-14, -1.9624e-14,\n -4.2082e-13, 3.1022e-13, 6.7305e-14, -7.9657e-14, 7.9149e-13,\n -1.1410e-12, -2.7759e-14, -6.7690e-15, 1.0310e-13, -8.0703e-13,\n 3.4302e-13, -1.9275e-13, -2.5672e-13, 6.6370e-13, 3.1384e-13,\n -1.4832e-13, 2.2517e-13, -1.7335e-13, 2.9186e-13, 4.4662e-13,\n 6.4742e-13, 2.1043e-13, -3.6018e-13, 7.5080e-13, -8.9517e-13,\n -2.0849e-13, 7.9746e-13, 3.4471e-13, 6.3106e-13, -1.7956e-13,\n 5.4644e-13, -7.7923e-13, 4.3613e-13, -8.7554e-13, 6.3603e-13,\n -6.2650e-13, -1.6227e-16, -1.5798e-13, 7.7171e-13, 3.9783e-13,\n 4.5966e-13, -9.8841e-14, -3.9413e-13, -1.2660e-12, 2.3918e-13,\n 6.0969e-13, 2.4147e-13, -7.0181e-13, -6.0506e-13, 4.1133e-13,\n 6.6239e-14, -4.7290e-13, 1.7199e-13, 5.8443e-13, 9.0646e-13,\n 2.7225e-13, -8.6141e-14, -1.1139e-12, -3.9646e-13, 2.4694e-13,\n -7.7058e-13, 3.9872e-13, 5.2493e-13, -1.3925e-13, -2.3309e-13,\n 8.2609e-13, -6.7291e-13, 2.5334e-13, 7.2357e-13, -6.8630e-13,\n 7.0476e-13, 3.9047e-13, -7.1275e-13, -3.3686e-13, 5.0605e-13,\n 1.9105e-13, -3.9037e-14, -8.5454e-14, -2.0162e-13, -3.9239e-13,\n -1.6935e-13, -5.3272e-14, -5.3568e-13, -7.6879e-13, 1.2508e-13,\n 8.9027e-13, 8.2377e-13, -9.0933e-13, 4.5381e-13, 7.3341e-13,\n 3.6452e-13, -1.2460e-13, -4.4166e-13, 4.0776e-13, 4.2835e-13,\n 1.9103e-13, 4.3215e-13, 2.4828e-13, 1.1938e-13, 3.2982e-13,\n 9.4426e-13, -4.5229e-13, -2.5834e-13, 5.0561e-13, 4.7972e-13,\n 3.3342e-13, 7.0462e-13, 4.9170e-13, 5.5721e-13, 3.1602e-14,\n -3.5180e-14, -2.3531e-13, 2.3520e-13, -5.8491e-13, -3.2401e-13,\n -4.3174e-13, -6.6467e-14, 5.1600e-13, -3.4000e-13, -4.7168e-13,\n 1.8456e-13, -3.1713e-13, -3.2027e-13, -7.9404e-13, -5.6500e-13,\n -9.7246e-13, 3.2258e-13, 3.9578e-13, -4.0062e-13, -2.7149e-13,\n 1.4497e-14, -1.7422e-13, -5.8991e-13, 5.5489e-13, 1.9948e-13,\n 2.5443e-13, -4.5637e-13, -5.3286e-13, -4.9673e-13, -3.9545e-13,\n 3.9304e-13, -4.0537e-13, 1.8799e-13, -3.1103e-13, -5.5774e-13,\n 4.4186e-13, -5.5433e-13, 4.1691e-13, 2.1201e-13, 7.6299e-13,\n 7.6578e-13, 8.0568e-13, 6.5055e-13, 3.9610e-13, -7.5509e-13,\n -5.6041e-14, 2.8699e-13, 2.5557e-13, -6.0070e-13, 9.8334e-14,\n 1.5622e-13, -8.6780e-14, 6.2580e-13, 6.0223e-13, -1.5790e-13,\n 5.0179e-13, -5.8897e-13, 5.8067e-13, -3.0469e-13, 5.3666e-13,\n -2.5905e-13, -1.3103e-12, 3.1954e-13, -1.1210e-12, -3.9877e-13,\n -1.0683e-13, 1.3966e-13, 8.1918e-14, 4.5553e-13, 1.6104e-13,\n -1.4358e-13, 3.9509e-13, -6.7990e-13, 4.8203e-13, 2.8992e-13,\n 3.0075e-14, 5.8631e-14, 1.2179e-13, -8.8447e-13, 8.4344e-13,\n -1.5473e-14, 1.5915e-13, -3.8171e-13, 5.6872e-13, 3.5804e-13,\n -4.5966e-13, 3.0214e-13, 8.1075e-14, 1.1650e-13, 4.4057e-13,\n -1.1473e-13, -5.9543e-13, -2.7713e-13, 8.4708e-13, -5.2741e-13,\n -6.8161e-13, 8.1451e-15, -6.9039e-13, 6.0889e-13, -6.0538e-13,\n 6.8194e-13, -2.9108e-13, 2.2154e-13, -7.0788e-14, -4.4559e-13,\n 4.8214e-13, -1.5250e-13, -5.0013e-13, -1.8954e-13, 8.0981e-13,\n -1.5899e-13, 2.2926e-13, 4.4133e-13, -2.4899e-13, 2.4843e-13,\n 8.6084e-13, -2.5433e-13, 2.8717e-13, -2.6852e-13, -1.1380e-12,\n 8.1126e-13, 1.6546e-13, -5.3793e-13, 2.4102e-13, -3.4178e-13,\n -3.0678e-13, -3.3045e-13, 5.4171e-13, -7.2365e-13, -9.1284e-14],\n device='cuda:0')", + "exp_avg_sq": "tensor([7.7647e-13, 5.6834e-13, 5.4036e-13, 5.9231e-13, 6.0626e-13, 7.8762e-13,\n 8.5793e-13, 1.0148e-12, 1.1946e-12, 2.9327e-13, 4.7946e-13, 3.3995e-13,\n 5.4900e-13, 5.7583e-13, 1.9755e-13, 5.8891e-13, 7.7435e-13, 4.0905e-13,\n 5.3269e-13, 6.9065e-13, 1.4402e-12, 9.0121e-13, 5.2608e-13, 1.2012e-12,\n 2.9720e-13, 8.4561e-13, 3.6031e-13, 6.5217e-13, 5.9970e-13, 7.7605e-13,\n 5.6281e-13, 1.5008e-12, 7.2997e-13, 5.5802e-13, 5.1930e-13, 1.3588e-12,\n 6.1579e-13, 7.0370e-13, 5.4381e-13, 1.3152e-13, 4.4562e-13, 1.3683e-12,\n 3.7834e-13, 3.9324e-13, 1.1927e-12, 9.2173e-13, 7.4089e-13, 3.3851e-13,\n 6.6058e-13, 4.9559e-13, 4.9174e-13, 1.1167e-12, 8.3627e-13, 9.7342e-13,\n 1.7533e-12, 3.9934e-13, 4.6873e-13, 3.5780e-13, 5.9506e-13, 7.8022e-13,\n 2.2265e-13, 4.2321e-13, 2.4149e-12, 6.1644e-13, 6.1313e-13, 1.1214e-12,\n 5.9120e-13, 1.9208e-13, 1.2672e-12, 5.4335e-13, 2.6153e-13, 3.0837e-13,\n 3.9679e-13, 2.9308e-13, 2.4439e-13, 1.1936e-12, 7.5929e-13, 3.2652e-13,\n 5.0573e-13, 5.1599e-13, 7.8063e-13, 8.2079e-13, 1.1816e-12, 7.3214e-13,\n 1.0355e-12, 6.6811e-13, 3.8396e-13, 3.9039e-13, 4.2527e-13, 1.6321e-12,\n 9.5714e-13, 1.5063e-12, 2.1900e-13, 6.7542e-13, 1.3807e-13, 2.4525e-13,\n 5.9686e-13, 6.7436e-13, 7.4499e-13, 1.2999e-12, 2.7831e-13, 1.8514e-13,\n 2.9312e-13, 8.9892e-13, 5.5927e-13, 4.4098e-13, 5.0067e-13, 2.8661e-13,\n 3.0952e-13, 6.1953e-13, 9.9577e-14, 4.1390e-13, 3.4474e-13, 7.4716e-13,\n 7.6031e-13, 4.4202e-13, 1.5360e-12, 2.9178e-13, 5.9827e-13, 6.7335e-13,\n 1.8861e-12, 6.2704e-13, 8.6104e-13, 5.2384e-13, 4.6765e-13, 2.9519e-13,\n 8.7175e-13, 8.9400e-13, 1.8651e-13, 7.3702e-13, 5.9440e-13, 4.8189e-13,\n 1.3789e-12, 1.8937e-13, 4.3210e-13, 9.4379e-13, 1.3556e-12, 8.9897e-13,\n 7.5990e-13, 3.6532e-13, 1.8798e-13, 1.1685e-12, 1.3483e-12, 8.7196e-13,\n 3.6097e-13, 2.2782e-13, 3.4056e-13, 4.1930e-13, 7.5314e-13, 3.3060e-13,\n 2.0417e-13, 3.3370e-13, 1.9682e-13, 1.4680e-12, 8.0558e-13, 3.4409e-13,\n 2.0933e-13, 1.5841e-12, 3.4665e-13, 6.2415e-13, 6.7195e-13, 1.1976e-12,\n 2.5470e-13, 6.1391e-13, 4.3526e-13, 7.7398e-13, 6.6802e-13, 1.5366e-12,\n 2.9579e-13, 3.4043e-13, 9.7861e-14, 1.6840e-12, 3.7831e-13, 2.1033e-12,\n 1.8425e-13, 3.7137e-13, 9.6358e-13, 3.3783e-13, 4.4457e-13, 3.2505e-13,\n 8.3285e-13, 6.6878e-13, 3.6941e-13, 1.7100e-12, 6.0496e-13, 3.3675e-13,\n 7.2324e-13, 6.0483e-13, 9.1825e-13, 4.2018e-13, 8.5778e-13, 1.1475e-12,\n 1.4556e-12, 8.7959e-13, 2.1917e-13, 5.4241e-13, 8.5221e-13, 6.4505e-13,\n 1.3137e-12, 5.0945e-13, 3.8361e-13, 3.2265e-13, 1.5857e-12, 4.4876e-13,\n 4.6843e-13, 3.5026e-13, 2.6885e-13, 3.3824e-13, 4.0660e-13, 2.6041e-13,\n 7.1710e-13, 3.4420e-13, 5.0192e-13, 6.3822e-13, 4.2995e-13, 1.3778e-12,\n 3.9630e-13, 1.1743e-12, 1.0631e-13, 6.0255e-13, 3.1600e-13, 9.0857e-13,\n 1.2487e-12, 1.0945e-12, 3.5791e-13, 1.3411e-12, 7.3797e-13, 5.5786e-13,\n 2.5146e-13, 2.4264e-13, 2.0451e-13, 1.6565e-13, 4.6625e-13, 1.8725e-12,\n 1.3157e-12, 4.6946e-13, 1.3648e-12, 2.7455e-13, 6.4443e-13, 2.4915e-13,\n 3.0076e-13, 5.3284e-13, 3.5298e-13, 1.2438e-12, 3.0876e-13, 9.6506e-13,\n 1.4708e-12, 8.8018e-13, 9.1415e-13, 2.3830e-13, 8.7106e-13, 9.5495e-13,\n 5.3361e-13, 7.7717e-13, 1.1753e-12, 2.5517e-13, 7.5181e-13, 9.8024e-13,\n 1.3454e-12, 6.4111e-13, 2.0644e-13, 7.0762e-13, 8.5088e-13, 5.4280e-13,\n 1.1583e-12, 8.9860e-13, 7.7184e-13, 6.1055e-13, 5.8022e-13, 1.1434e-12,\n 1.9990e-13, 7.1994e-13, 2.9003e-12, 2.7583e-13, 1.6478e-12, 8.7390e-13,\n 2.9868e-13, 1.6020e-12, 3.5852e-13, 1.0163e-12, 1.0589e-12, 4.7044e-13,\n 1.2135e-12, 1.3472e-12, 3.8233e-13, 1.2003e-12, 1.6775e-12, 3.9115e-13,\n 1.2311e-13, 3.4936e-13, 7.6509e-13, 2.6452e-13, 1.0021e-12, 8.2305e-13,\n 3.7311e-13, 4.9310e-13, 3.4726e-13, 1.0240e-12, 1.9659e-13, 5.8613e-13,\n 8.5070e-13, 8.6948e-13, 4.8242e-13, 5.2394e-13, 2.0686e-13, 1.6114e-13,\n 1.9196e-12, 4.5724e-13, 7.6945e-13, 1.8220e-13, 1.5451e-12, 1.3684e-12,\n 9.2135e-13, 9.4274e-13, 1.1046e-12, 5.6201e-13, 7.6152e-13, 8.4045e-13,\n 2.6447e-13, 3.8453e-13, 6.9185e-13, 5.1415e-13, 1.0036e-12, 5.9741e-13,\n 7.8606e-13, 1.1035e-12, 1.2585e-12, 9.9195e-13, 1.1356e-12, 4.4000e-13,\n 6.4159e-13, 1.3449e-12, 3.4570e-13, 1.9376e-13, 2.9149e-13, 4.1970e-13,\n 3.1910e-13, 5.0356e-13, 8.0754e-13, 9.5194e-13, 6.9613e-13, 3.0896e-13,\n 9.9854e-13, 8.2723e-13, 2.3305e-13, 3.8137e-13, 3.4269e-13, 1.0728e-12,\n 1.4228e-12, 2.1344e-13, 5.5430e-13, 5.3658e-13, 7.0230e-13, 2.5648e-13,\n 8.1196e-13, 4.3116e-13, 2.3243e-13, 3.6698e-13, 4.9030e-13, 5.1171e-13,\n 2.5113e-13, 6.8941e-13, 3.6433e-13, 9.4454e-13, 5.9830e-13, 1.4907e-12,\n 1.3048e-12, 4.7296e-13, 4.4003e-13, 5.4236e-13, 1.1978e-12, 1.4127e-12,\n 7.1098e-13, 3.2438e-13, 3.4551e-13, 1.9872e-13, 4.5466e-13, 6.3042e-13,\n 4.9946e-13, 8.6432e-13, 3.2016e-13, 2.5844e-13, 3.2551e-13, 4.6852e-13,\n 2.0013e-13, 1.0746e-12, 7.6663e-13, 5.3580e-13, 7.8825e-13, 1.6534e-12,\n 4.0468e-13, 3.3819e-13, 2.3851e-13, 1.5071e-12, 7.9909e-13, 7.5613e-13,\n 2.1369e-13, 7.8868e-13, 2.8326e-13, 7.4205e-13, 4.6303e-13, 3.8609e-13,\n 3.6521e-13, 6.1374e-13, 7.5311e-13, 1.6472e-12, 2.9284e-13, 9.1157e-13,\n 6.9187e-13, 6.7021e-13, 3.8579e-13, 2.3135e-13, 5.7034e-13, 9.5008e-13,\n 2.0109e-13, 5.9338e-13, 1.2748e-12, 2.9163e-13, 8.7142e-13, 6.6380e-13,\n 3.5527e-13, 1.2882e-12, 4.3887e-13, 1.7031e-13, 5.9493e-13, 3.5871e-13,\n 7.3838e-13, 4.9022e-13, 6.8701e-13, 6.6711e-13, 6.2541e-13, 2.4871e-13,\n 3.6898e-13, 4.3309e-13, 7.2107e-13, 3.9861e-13, 3.6630e-13, 5.8675e-13,\n 9.6270e-13, 4.6516e-13, 1.5646e-12, 1.5182e-12, 2.4396e-13, 1.9315e-12,\n 2.4494e-13, 5.6778e-13, 3.0348e-13, 1.0276e-12, 3.7617e-13, 5.2448e-13,\n 2.4567e-12, 4.0967e-13, 4.5941e-13, 5.8591e-13, 1.1623e-12, 4.3993e-13,\n 7.2262e-13, 6.0075e-13, 3.7720e-13, 3.4471e-13, 3.9703e-13, 2.4661e-13,\n 1.1252e-12, 1.3466e-13, 1.0948e-12, 1.1403e-12, 5.7530e-13, 2.5263e-13,\n 3.4225e-13, 1.9835e-12, 4.6655e-13, 3.3471e-13, 2.0095e-13, 4.7639e-13,\n 8.1420e-13, 6.1742e-13, 5.5516e-13, 9.9761e-13, 1.9311e-13, 6.8428e-13,\n 5.8965e-13, 1.0174e-12, 1.2816e-13, 2.6311e-13, 4.0425e-13, 6.2923e-13,\n 1.1012e-12, 4.0908e-13, 4.0741e-13, 2.9237e-13, 1.7558e-13, 7.2377e-13,\n 5.3921e-13, 4.1535e-13, 3.7354e-13, 6.9549e-13, 2.4201e-13, 3.1307e-13,\n 6.2413e-13, 4.6227e-13, 1.1501e-12, 3.8054e-13, 5.8356e-13, 9.8892e-13,\n 4.4224e-13, 1.0456e-12, 1.0472e-12, 2.5256e-13, 7.8038e-13, 4.4121e-13,\n 2.6433e-13, 7.7655e-13, 4.1675e-13, 9.3182e-13, 1.3287e-12, 5.9860e-13,\n 4.9419e-13, 3.5355e-13, 2.6163e-13, 1.9309e-13, 3.0740e-13, 1.1528e-12,\n 2.1162e-13, 4.0722e-13, 9.7608e-13, 4.5324e-13, 1.2001e-12, 6.8556e-13,\n 6.0218e-13, 1.8327e-13, 2.2654e-13, 3.0400e-13, 6.3094e-13, 2.0663e-13,\n 1.2153e-12, 8.6188e-13, 6.2502e-13, 8.9362e-13, 2.3108e-13, 3.4732e-13,\n 1.5928e-13, 5.5460e-13, 1.0997e-12, 5.2751e-13, 6.6339e-13, 5.7638e-13,\n 1.2937e-12, 7.5321e-13, 6.5595e-13, 1.5872e-12, 4.4162e-13, 1.3667e-13,\n 5.5301e-13, 4.5369e-13, 5.4551e-13, 3.1465e-13, 2.9076e-13, 2.2856e-13,\n 3.7011e-13, 5.4485e-13, 2.0397e-13, 3.0900e-13, 6.5859e-13, 3.9005e-13,\n 6.3255e-13, 3.4383e-13, 7.1384e-13, 6.4728e-13, 3.0796e-13, 1.2122e-12,\n 1.1542e-12, 1.5520e-12, 3.2192e-13, 3.7824e-13, 1.7555e-13, 1.0080e-12,\n 8.4858e-13, 2.3501e-13, 2.0044e-13, 3.4429e-13, 5.4693e-13, 6.5277e-13,\n 8.0113e-13, 4.0613e-13, 9.4091e-13, 4.4431e-13, 2.1031e-13, 3.6564e-13,\n 3.5893e-13, 7.4431e-13, 4.2451e-13, 9.1185e-13, 5.0855e-13, 9.3216e-13,\n 1.0769e-12, 1.3216e-13, 1.0076e-12, 1.8779e-13, 2.3824e-13, 9.3733e-13,\n 7.1045e-13, 3.5945e-13, 2.9749e-13, 3.3680e-13, 3.7181e-13, 1.8185e-12,\n 4.1121e-13, 8.2296e-13, 1.3484e-12, 1.0038e-12, 1.2955e-12, 2.4305e-13,\n 1.9256e-13, 3.0090e-13, 6.3997e-13, 2.1804e-12, 5.4389e-13, 3.0789e-13,\n 1.9446e-13, 3.7737e-13, 5.3926e-13, 4.3962e-13, 3.2668e-13, 4.8202e-13,\n 7.8427e-13, 5.5965e-13, 9.2529e-13, 2.8144e-13, 1.0269e-12, 8.8206e-13,\n 1.1451e-12, 6.6409e-13, 2.4258e-13, 6.6707e-13, 5.4539e-13, 9.2269e-13,\n 4.2756e-13, 2.9001e-13, 1.4193e-12, 1.1164e-13], device='cuda:0')" + }, + "66": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-2.6697e-12, -1.6299e-12, -1.5347e-12, ..., 2.2307e-12,\n -3.0152e-12, -8.8597e-14],\n [-2.4652e-19, -2.6815e-20, 4.3440e-19, ..., 2.4418e-18,\n -7.6743e-19, -7.1956e-19],\n [-4.7352e-12, -2.9991e-12, -2.3505e-12, ..., 4.6482e-12,\n -5.9591e-12, -1.9718e-13],\n ...,\n [ 3.3350e-16, 2.4155e-16, 1.8928e-16, ..., -2.8489e-16,\n 4.0921e-16, 1.5262e-17],\n [ 1.7258e-14, 1.2153e-14, 1.0016e-14, ..., -1.3674e-14,\n 2.0208e-14, 4.4810e-16],\n [ 7.6828e-15, 5.6986e-15, 4.5228e-15, ..., -6.2023e-15,\n 9.2546e-15, 9.0171e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7373e-11, 6.5846e-12, 4.4813e-12, ..., 5.5667e-12, 1.7509e-11,\n 3.4453e-12],\n [9.3712e-13, 2.1308e-13, 2.2173e-13, ..., 8.3420e-14, 4.4027e-13,\n 2.8247e-13],\n [2.7534e-11, 3.1779e-12, 4.8939e-12, ..., 6.3843e-12, 1.8649e-11,\n 1.4279e-11],\n ...,\n [1.8862e-13, 3.2093e-13, 1.8644e-13, ..., 6.4603e-14, 5.2881e-14,\n 2.1308e-13],\n [6.8750e-12, 3.0932e-13, 2.4614e-12, ..., 2.1901e-13, 1.9803e-13,\n 7.6118e-12],\n [1.7729e-12, 4.8163e-13, 3.2564e-13, ..., 4.4392e-13, 6.7580e-14,\n 2.1499e-12]], device='cuda:0')" + }, + "67": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.3879e-12, -6.4774e-20, 5.4665e-12, -2.4906e-17, -3.1416e-15,\n -3.6297e-16, -1.8212e-14, -8.8474e-15], device='cuda:0')", + "exp_avg_sq": "tensor([7.0420e-11, 1.3012e-11, 1.0142e-10, 3.4384e-12, 2.4192e-11, 6.7460e-12,\n 3.7940e-11, 1.7960e-11], device='cuda:0')" + }, + "68": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-2.6791e-12, -1.6356e-12, -1.5402e-12, ..., 2.2387e-12,\n -3.0259e-12, -8.8899e-14],\n [-2.4922e-19, -2.8356e-20, 4.3506e-19, ..., 2.4530e-18,\n -7.7287e-19, -7.2250e-19],\n [-4.7488e-12, -3.0076e-12, -2.3572e-12, ..., 4.6617e-12,\n -5.9763e-12, -1.9775e-13],\n ...,\n [ 3.3472e-16, 2.4244e-16, 1.8997e-16, ..., -2.8595e-16,\n 4.1072e-16, 1.5321e-17],\n [ 1.7319e-14, 1.2196e-14, 1.0051e-14, ..., -1.3723e-14,\n 2.0280e-14, 4.4966e-16],\n [ 7.7089e-15, 5.7178e-15, 4.5382e-15, ..., -6.2232e-15,\n 9.2858e-15, 9.0498e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7425e-11, 6.5817e-12, 4.4887e-12, ..., 5.5837e-12, 1.7566e-11,\n 3.4482e-12],\n [9.3770e-13, 2.1297e-13, 2.2172e-13, ..., 8.3305e-14, 4.4065e-13,\n 2.8223e-13],\n [2.7543e-11, 3.1849e-12, 4.8979e-12, ..., 6.3976e-12, 1.8704e-11,\n 1.4236e-11],\n ...,\n [1.8807e-13, 3.2011e-13, 1.8610e-13, ..., 6.4469e-14, 5.2783e-14,\n 2.1233e-13],\n [6.8484e-12, 3.0836e-13, 2.4522e-12, ..., 2.1817e-13, 1.9751e-13,\n 7.5822e-12],\n [1.7662e-12, 4.7993e-13, 3.2441e-13, ..., 4.4227e-13, 6.7400e-14,\n 2.1411e-12]], device='cuda:0')" + }, + "69": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.3964e-12, -6.5005e-20, 5.4860e-12, -2.4995e-17, -3.1528e-15,\n -3.6426e-16, -1.8276e-14, -8.8789e-15], device='cuda:0')", + "exp_avg_sq": "tensor([7.0431e-11, 1.3013e-11, 1.0128e-10, 3.4308e-12, 2.4108e-11, 6.7317e-12,\n 3.7802e-11, 1.7901e-11], device='cuda:0')" + }, + "70": { + "step": "tensor(8764.)", + "exp_avg": "tensor([[-2.6633e-12, -1.6260e-12, -1.5311e-12, ..., 2.2254e-12,\n -3.0079e-12, -8.8434e-14],\n [-2.5005e-19, -2.9761e-20, 4.3087e-19, ..., 2.4403e-18,\n -7.7064e-19, -7.1835e-19],\n [-4.7245e-12, -2.9923e-12, -2.3452e-12, ..., 4.6377e-12,\n -5.9457e-12, -1.9674e-13],\n ...,\n [ 3.3291e-16, 2.4112e-16, 1.8893e-16, ..., -2.8440e-16,\n 4.0848e-16, 1.5239e-17],\n [ 1.7226e-14, 1.2130e-14, 9.9970e-15, ..., -1.3649e-14,\n 2.0171e-14, 4.4712e-16],\n [ 7.6663e-15, 5.6862e-15, 4.5131e-15, ..., -6.1892e-15,\n 9.2347e-15, 9.0028e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.7350e-11, 6.5700e-12, 4.4745e-12, ..., 5.5591e-12, 1.7486e-11,\n 3.4388e-12],\n [9.3565e-13, 2.1273e-13, 2.2154e-13, ..., 8.3279e-14, 4.3956e-13,\n 2.8191e-13],\n [2.7478e-11, 3.1728e-12, 4.8846e-12, ..., 6.3737e-12, 1.8622e-11,\n 1.4240e-11],\n ...,\n [1.8812e-13, 3.2002e-13, 1.8609e-13, ..., 6.4462e-14, 5.2744e-14,\n 2.1245e-13],\n [6.8556e-12, 3.0845e-13, 2.4546e-12, ..., 2.1838e-13, 1.9752e-13,\n 7.5902e-12],\n [1.7678e-12, 4.8016e-13, 3.2465e-13, ..., 4.4259e-13, 6.7414e-14,\n 2.1434e-12]], device='cuda:0')" + }, + "71": { + "step": "tensor(8764.)", + "exp_avg": "tensor([ 2.3832e-12, -6.4645e-20, 5.4557e-12, -2.4857e-17, -3.1354e-15,\n -3.6225e-16, -1.8176e-14, -8.8300e-15], device='cuda:0')", + "exp_avg_sq": "tensor([7.0286e-11, 1.2994e-11, 1.0119e-10, 3.4302e-12, 2.4115e-11, 6.7304e-12,\n 3.7832e-11, 1.7909e-11], device='cuda:0')" + } + }, + "param_groups": [ + { + "lr": 0.00020690126647990973, + "name": "shared", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 0, + 1 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_384", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 2, + 3, + 4 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_512", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 5, + 6, + 7 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_768", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 8, + 9, + 10 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_1024", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 11, + 12, + 13 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_1280", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 14, + 15, + 16 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_1536", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 17, + 18, + 19 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_1792", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 20, + 21, + 22 + ] + }, + { + "lr": 0.00020690126647990973, + "name": "scale_2048", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.001, + "params": [ + 23, + 24, + 25 + ] + }, + { + "lr": 0.00010384757955302797, + "name": "fusion", + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0005, + "params": [ + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71 + ] + } + ] + }, + "scheduler_state_dict": { + "T_0": 10, + "T_i": 10, + "T_mult": 2, + "eta_min": 1e-06, + "T_cur": 7, + "base_lrs": [ + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.001, + 0.0005 + ], + "last_epoch": 7, + "_step_count": 0, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00020690126647990973, + 0.00010384757955302797 + ] + }, + "metrics": { + "best_val_acc": 84.586, + "best_epoch": 6, + "scale_accuracies": { + "384": 84.672, + "512": 84.618, + "768": 84.586, + "1024": 84.368, + "1280": 84.366, + "1536": 84.46, + "1792": 84.254, + "2048": 84.364 + }, + "training_history": { + "epochs": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7 + ], + "train_loss": [ + 2.090034657678665, + 1.2265579131083748, + 1.0678428365304447, + 0.9437485171583133, + 0.8316704105740538, + 0.7262678437244398, + 0.6332142021233281 + ], + "train_acc": [ + 81.03081019102116, + 86.79532020415762, + 88.322443522195, + 89.53789786967663, + 90.7392244726878, + 91.92845273098668, + 93.03353895315755 + ], + "val_acc": [ + 83.714, + 84.042, + 84.372, + 84.414, + 84.578, + 84.512, + 84.586 + ], + "scale_accs": { + "384": [ + 83.52, + 83.85, + 84.262, + 84.314, + 84.636, + 84.464, + 84.672 + ], + "512": [ + 83.54, + 83.942, + 84.382, + 84.448, + 84.67, + 84.622, + 84.618 + ], + "768": [ + 83.714, + 84.042, + 84.372, + 84.414, + 84.578, + 84.512, + 84.586 + ], + "1024": [ + 83.484, + 84.12, + 84.464, + 84.44, + 84.402, + 84.508, + 84.368 + ], + "1280": [ + 83.63, + 83.998, + 84.362, + 84.276, + 84.516, + 84.346, + 84.366 + ], + "1536": [ + 83.592, + 84.212, + 84.324, + 84.37, + 84.41, + 84.436, + 84.46 + ], + "1792": [ + 83.622, + 83.96, + 84.282, + 84.326, + 84.406, + 84.276, + 84.254 + ], + "2048": [ + 83.664, + 84.082, + 84.258, + 84.416, + 84.38, + 84.396, + 84.364 + ] + }, + "lr": [ + 0.0009755527298894294, + 0.0009046039886902864, + 0.0007940987335200904, + 0.0006548539886902864, + 0.0005005000000000001, + 0.0003461460113097139, + 0.00020690126647990973 + ] + } + }, + "train_config": { + "name": "david_training", + "run_id": "20251012_161107", + "dataset_name": "AbstractPhil/imagenet-clip-features-orderly", + "model_variant": "clip_vit_laion_bigg14", + "num_classes": 1000, + "preset": "clip_vit_bigg14", + "custom_config_path": null, + "num_classes_override": null, + "use_belly_override": null, + "belly_expand_override": null, + "progressive_training_override": false, + "scale_warmup_epochs_override": null, + "num_epochs": 10, + "batch_size": 1024, + "learning_rate": 0.001, + "weight_decay": 1e-05, + "warmup_epochs": 3, + "use_rose_loss": true, + "rose_initial_weight": 0.1, + "rose_max_weight": 0.5, + "rose_weight_schedule": "adaptive", + "use_cayley_loss": false, + "cayley_weight": 0.001, + "scale_loss_balance": null, + "use_mixed_precision": false, + "gradient_clip": 10.0, + "scheduler_type": "cosine_restarts", + "min_lr": 1e-06, + "freeze_strategy": "none", + "freeze_threshold": 90.0, + "unfreeze_on_plateau": true, + "patience": 10, + "track_gradients": true, + "gradient_scale_threshold": 1e-05, + "gradient_scale_multiplier": 10.0, + "log_interval": 50, + "val_interval": 1, + "save_interval": 5, + "log_fusion_weights": true, + "log_loss_components": true, + "save_format": "safetensors", + "hf_repo": "AbstractPhil/gated-david", + "upload_to_hub": true, + "base_dir": "./david_training", + "num_workers": 10, + "pin_memory": true, + "prefetch_factor": 4, + "persistent_workers": true + } +} \ No newline at end of file