diff --git "a/weights/best_model_metadata.json" "b/weights/best_model_metadata.json" --- "a/weights/best_model_metadata.json" +++ "b/weights/best_model_metadata.json" @@ -1,36 +1,196 @@ { - "epoch": 0, + "epoch": 1, "optimizer_state_dict": { "state": { "0": { - "step": "tensor(1252.)", - "exp_avg": "tensor([[ 7.6931e-04, -3.5975e-04, -2.7454e-04, ..., 3.4282e-04,\n 1.6405e-04, -2.7417e-04],\n [-1.1996e-04, -1.1731e-04, -9.9866e-05, ..., 1.6243e-04,\n 2.2704e-04, 5.8436e-05],\n [-2.3594e-05, -1.2594e-04, 1.5107e-04, ..., -3.0109e-04,\n -7.9584e-05, 6.9697e-05],\n ...,\n [-1.2359e-04, -1.4653e-05, 2.1027e-04, ..., -1.2977e-04,\n 1.0232e-04, 1.3696e-04],\n [ 1.5723e-04, 9.8239e-05, -1.2136e-04, ..., 1.6422e-04,\n 1.6617e-04, -3.1527e-04],\n [ 3.0708e-04, -7.8846e-04, 5.5491e-05, ..., -8.6530e-05,\n 2.0349e-04, -1.3880e-04]], device='cuda:0')", - "exp_avg_sq": "tensor([[7.4118e-07, 1.0833e-06, 5.3915e-07, ..., 5.2782e-07, 4.1944e-07,\n 3.6632e-07],\n [3.1076e-07, 3.0220e-07, 3.1742e-07, ..., 2.1258e-07, 1.9540e-07,\n 1.6665e-07],\n [6.6361e-07, 7.0572e-07, 4.0617e-07, ..., 4.1036e-07, 3.5275e-07,\n 2.9789e-07],\n ...,\n [4.7457e-07, 3.5343e-07, 3.7321e-07, ..., 3.1882e-07, 2.7955e-07,\n 2.5810e-07],\n [5.7871e-07, 5.5442e-07, 4.4099e-07, ..., 3.3348e-07, 3.2790e-07,\n 2.9927e-07],\n [5.5174e-07, 5.0187e-07, 2.8968e-07, ..., 3.2638e-07, 3.1291e-07,\n 2.3588e-07]], device='cuda:0')" + "step": "tensor(2504.)", + "exp_avg": "tensor([[ 8.7452e-05, -2.1733e-04, -1.3883e-04, ..., 9.8448e-06,\n -5.7181e-05, 6.8193e-05],\n [ 3.9587e-05, 3.4732e-05, 2.2891e-05, ..., 4.8062e-06,\n 1.1224e-04, 4.1794e-05],\n [-3.3535e-05, -3.6872e-05, -9.1059e-05, ..., -1.3798e-05,\n -2.7946e-05, 5.1825e-05],\n ...,\n [ 1.8504e-05, -6.8543e-05, 3.1895e-04, ..., 1.7729e-04,\n 1.1905e-04, 7.3752e-05],\n [-7.4677e-06, -9.6732e-05, 1.2372e-04, ..., -1.1602e-04,\n -1.1712e-04, 5.8511e-05],\n [-5.9348e-04, -6.2306e-04, 3.4747e-04, ..., -5.9401e-04,\n -5.4200e-04, 1.1022e-04]], device='cuda:0')", + "exp_avg_sq": "tensor([[6.5301e-07, 1.0733e-06, 5.8385e-07, ..., 4.9240e-07, 4.0495e-07,\n 3.3179e-07],\n [3.0402e-07, 3.1917e-07, 3.0831e-07, ..., 2.0204e-07, 1.8485e-07,\n 1.5767e-07],\n [5.1834e-07, 5.4625e-07, 3.3273e-07, ..., 3.0887e-07, 2.9542e-07,\n 2.4476e-07],\n ...,\n [5.0262e-07, 3.5524e-07, 3.3082e-07, ..., 3.2476e-07, 2.6730e-07,\n 2.5890e-07],\n [4.4557e-07, 4.0738e-07, 3.2369e-07, ..., 2.5348e-07, 2.3750e-07,\n 2.1727e-07],\n [5.2584e-07, 5.1742e-07, 2.5882e-07, ..., 3.0717e-07, 3.1134e-07,\n 2.2695e-07]], device='cuda:0')" }, "1": { - "step": "tensor(1252.)", - "exp_avg": "tensor([ 0.0155, -0.0063, -0.0059, ..., -0.0034, 0.0076, 0.0054],\n device='cuda:0')", - "exp_avg_sq": "tensor([0.0012, 0.0005, 0.0010, ..., 0.0009, 0.0010, 0.0007], device='cuda:0')" + "step": "tensor(2504.)", + "exp_avg": "tensor([ 0.0075, -0.0002, -0.0005, ..., 0.0001, -0.0030, -0.0211],\n device='cuda:0')", + "exp_avg_sq": "tensor([0.0010, 0.0005, 0.0008, ..., 0.0007, 0.0006, 0.0007], device='cuda:0')" }, "2": { - "step": "tensor(1252.)", - "exp_avg": "tensor([[ 1.3985e-05, -5.8933e-05, 1.0128e-04, ..., 2.2678e-04,\n -1.4801e-04, -2.0768e-05],\n [ 4.3151e-04, 1.1516e-04, -8.9230e-05, ..., 4.8063e-05,\n -2.3465e-04, 4.5604e-06],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -3.1455e-32,\n 9.5699e-32, -5.6052e-45],\n ...,\n [ 6.3424e-09, -1.6865e-05, 2.2038e-07, ..., -4.1170e-07,\n -9.3469e-06, -1.4808e-07],\n [ 1.1432e-05, 3.8929e-05, -3.2332e-05, ..., 7.8134e-07,\n 4.3614e-05, -1.0174e-05],\n [-9.4698e-07, -2.8017e-07, -1.3962e-07, ..., -1.0875e-04,\n -4.0648e-07, -4.5041e-06]], device='cuda:0')", - "exp_avg_sq": "tensor([[1.0298e-06, 4.7161e-07, 2.3258e-07, ..., 1.3600e-06, 7.8774e-07,\n 6.1178e-07],\n [1.9419e-06, 8.5299e-07, 6.2341e-07, ..., 1.7004e-06, 8.0653e-07,\n 6.4486e-07],\n [1.7955e-10, 7.0238e-10, 3.7342e-11, ..., 5.1371e-10, 1.6562e-10,\n 1.7251e-10],\n ...,\n [7.4679e-11, 4.4698e-09, 6.7743e-11, ..., 3.5092e-10, 5.4696e-09,\n 2.4203e-10],\n [1.2516e-09, 6.6520e-08, 8.9164e-09, ..., 7.1810e-09, 1.6729e-08,\n 4.2439e-09],\n [1.1515e-08, 7.9548e-09, 6.6302e-09, ..., 7.8305e-07, 3.6041e-09,\n 6.1598e-08]], device='cuda:0')" + "step": "tensor(2504.)", + "exp_avg": "tensor([[ 4.2469e-05, -7.2644e-05, -1.1952e-05, ..., 4.7904e-05,\n 7.3089e-05, 3.2043e-05],\n [ 2.6812e-04, -2.8175e-04, 6.2112e-05, ..., -3.1649e-04,\n 1.4217e-05, -6.0623e-04],\n [ 5.6052e-45, 5.6052e-45, 5.6052e-45, ..., -5.6052e-45,\n 5.6052e-45, -5.6052e-45],\n ...,\n [-6.2003e-08, -4.5649e-06, -2.5983e-07, ..., 6.3899e-06,\n 1.6855e-06, 3.8471e-06],\n [ 9.1794e-06, -1.4863e-06, -6.9139e-06, ..., -1.6460e-05,\n 1.1438e-04, 1.4669e-06],\n [ 5.6084e-06, 3.0001e-06, -8.2603e-07, ..., -4.0520e-04,\n 2.1599e-05, -5.6674e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[9.2716e-07, 5.9824e-07, 1.1594e-07, ..., 1.0178e-06, 3.5610e-07,\n 3.5609e-07],\n [1.2661e-06, 5.8140e-07, 3.5091e-07, ..., 1.2685e-06, 3.5320e-07,\n 4.8524e-07],\n [5.1307e-11, 2.0071e-10, 1.0671e-11, ..., 1.4680e-10, 4.7328e-11,\n 4.9297e-11],\n ...,\n [9.2620e-09, 7.7398e-09, 4.9677e-10, ..., 1.0957e-08, 4.7122e-09,\n 7.6998e-10],\n [8.7455e-09, 3.1903e-08, 4.4403e-08, ..., 5.7507e-09, 3.9395e-08,\n 1.6215e-08],\n [1.5398e-08, 3.5247e-09, 2.2004e-09, ..., 5.9095e-07, 1.8270e-09,\n 3.3412e-08]], device='cuda:0')" }, "3": { - "step": "tensor(1252.)", - "exp_avg": "tensor([ 2.3875e-02, -4.3837e-02, 9.6032e-26, 4.2214e-02, 2.2210e-02,\n 3.3747e-02, 1.7797e-02, -7.0004e-03, -2.4879e-02, 2.7619e-02,\n -7.5551e-03, 1.7134e-02, 2.0256e-02, -2.3314e-03, -7.4220e-02,\n -4.7382e-03, 1.4119e-02, -1.1545e-02, -8.9664e-03, 2.9760e-02,\n 2.9607e-03, -2.0514e-02, -7.0222e-03, -4.5160e-02, -2.3961e-02,\n 2.8237e-04, -1.7209e-02, 2.0788e-02, -7.2040e-03, 3.1665e-02,\n 5.6030e-03, 3.4553e-02, 3.6798e-02, -7.2686e-03, 3.0085e-02,\n 7.4137e-03, -6.1774e-03, 5.0785e-02, 4.7387e-03, 3.7325e-02,\n 2.7934e-02, -3.6341e-02, -2.1846e-02, 5.6052e-45, -2.4211e-02,\n 2.2644e-03, 5.5101e-02, 1.4802e-02, -5.4617e-03, -1.9478e-02,\n -6.5484e-03, -1.6414e-02, -4.0664e-04, 4.5538e-02, 5.8758e-03,\n 2.9059e-02, -1.2425e-02, 3.5538e-03, 4.1299e-03, 9.5794e-03,\n -4.0616e-02, 4.8972e-03, 2.5462e-02, 7.2303e-02, 1.5787e-03,\n 4.5761e-02, 1.9496e-02, 4.2323e-02, 3.9632e-02, -1.0082e-03,\n 5.3938e-03, 3.3493e-05, -1.3032e-02, -3.8153e-02, 1.5953e-02,\n 4.4613e-06, 7.0803e-03, -3.0465e-04, -9.4197e-03, -2.4202e-02,\n 9.6786e-02, -7.4001e-03, -2.9757e-02, 2.0539e-02, -2.8599e-02,\n 2.9271e-02, -1.7663e-03, 7.1197e-03, 2.4915e-02, 1.2627e-32,\n -3.4740e-02, 8.6782e-03, -2.3629e-09, 1.6301e-29, 1.5997e-02,\n 4.4684e-02, 3.8713e-04, -1.4058e-02, 2.3174e-02, -1.3123e-02,\n 9.1376e-03, -1.8078e-02, 1.6822e-02, 1.1673e-05, 1.4093e-02,\n 3.1474e-02, -1.2471e-02, -3.0086e-02, -2.3609e-02, 7.3296e-03,\n 5.4473e-36, 1.4037e-02, -1.1540e-02, 3.6438e-03, 2.6305e-03,\n -1.2727e-02, 2.7998e-03, -1.4901e-02, 2.1351e-02, -4.3767e-15,\n 1.8248e-31, 6.4915e-02, 1.7753e-02, 4.6665e-02, 1.1710e-02,\n -1.0307e-02, 2.7988e-02, 2.5342e-02, 2.9874e-02, -3.5108e-02,\n -2.3540e-03, -6.9490e-02, -7.4078e-03, 2.1446e-02, -1.5282e-02,\n -9.1815e-03, -5.9992e-03, -1.6854e-02, -1.2549e-02, -2.3482e-02,\n -2.0157e-02, 2.2761e-19, 5.0847e-02, 2.5370e-02, -9.5745e-03,\n 2.6735e-02, 6.3855e-03, 2.5707e-02, -3.6340e-02, -5.3170e-02,\n 2.5905e-02, -2.7917e-04, 5.1961e-02, -8.5205e-03, -2.0016e-02,\n 7.9647e-03, 2.6824e-28, -1.5696e-02, 1.9396e-02, 9.1115e-03,\n -5.6675e-02, 3.7877e-02, -2.4820e-03, 3.0127e-02, 3.7650e-02,\n 1.3167e-02, -6.0690e-02, 1.5634e-03, 2.0013e-02, -2.8012e-03,\n 2.7729e-02, 6.1169e-03, 7.8048e-04, 5.2743e-03, -3.7234e-02,\n -6.2202e-03, -4.8082e-03, 3.5392e-02, 2.1980e-02, 6.6747e-03,\n -5.1266e-04, 4.6985e-02, 5.6052e-45, 7.1385e-05, 3.2047e-03,\n -3.6594e-02, 1.8168e-02, -6.2917e-02, -3.2929e-02, -5.6694e-03,\n 1.9831e-02, -8.2062e-04, 1.5245e-02, -3.5059e-02, -7.1714e-03,\n 3.2220e-03, -2.1203e-04, -6.0919e-03, 1.4150e-02, -1.7816e-02,\n -1.6064e-02, 1.0769e-02, -2.6079e-02, -9.8037e-03, -3.2816e-02,\n -5.2574e-02, -4.7450e-03, 1.9516e-02, 3.4608e-03, 1.7244e-02,\n 2.6142e-13, 6.6794e-04, 2.7124e-02, -3.4816e-02, -3.1702e-02,\n 3.3221e-03, 3.8579e-02, -3.2285e-02, 1.9722e-02, 1.8849e-02,\n -3.3065e-02, 5.6052e-45, -7.1891e-03, -2.5881e-02, 6.2611e-03,\n -1.4979e-02, -6.4840e-03, 3.5860e-02, -2.7866e-02, -1.3252e-02,\n 2.1320e-02, 1.7814e-02, -3.2913e-02, -2.2322e-02, 2.0048e-03,\n 9.2803e-03, -8.2572e-03, -5.2975e-03, 7.6453e-02, 4.8804e-02,\n -2.1919e-02, 5.6761e-03, 6.3496e-02, -2.8889e-02, 1.3841e-02,\n 8.5695e-03, -1.5521e-02, 4.9531e-02, -4.1589e-02, -1.9676e-02,\n -5.5857e-03, -2.3588e-03, -1.4790e-02, -1.3780e-03, -6.3026e-03,\n -1.0188e-02, -3.3702e-02, 1.9226e-42, 2.2306e-02, 2.6239e-02,\n 4.8861e-26, -5.2627e-03, 2.5603e-06, 7.0623e-03, -4.3391e-03,\n 6.8443e-03, -1.6604e-02, -1.0740e-02, -1.5329e-02, 2.6949e-03,\n -8.0152e-03, -5.2252e-03, 8.3405e-03, 4.8676e-03, 2.9752e-02,\n 2.1431e-02, 5.0216e-02, 2.7263e-02, 8.0592e-03, 8.6727e-35,\n -2.2584e-02, -2.4758e-02, -2.9040e-02, -1.3793e-02, 1.8160e-03,\n -6.5728e-04, -4.3327e-02, 5.8940e-03, -1.1455e-02, -1.5254e-02,\n -1.3119e-02, -2.3658e-02, -6.1895e-03, 2.2715e-02, 2.3466e-02,\n 9.7893e-03, 3.0946e-02, -5.9481e-02, 6.9836e-09, 3.0786e-02,\n 2.9153e-02, -2.5735e-02, -7.1205e-03, -2.9654e-03, 3.6341e-02,\n -1.8624e-02, 2.3698e-02, -4.9125e-03, 1.7921e-02, -4.5526e-03,\n 2.6053e-02, -3.2306e-02, 7.3569e-02, -2.9119e-02, -8.4770e-03,\n 2.6320e-02, -3.1927e-02, 8.3118e-03, 2.1609e-02, 8.7364e-03,\n -6.7107e-03, 1.2142e-02, 8.1370e-03, -3.0566e-02, -1.4636e-02,\n 1.1678e-30, 6.2592e-05, 7.6997e-03, -5.0519e-02, 5.8788e-03,\n 3.3712e-02, 1.9211e-11, -2.0315e-02, 1.6669e-02, -4.9647e-03,\n -8.5470e-03, -5.1239e-03, 2.8877e-14, 8.1417e-08, -3.9018e-03,\n -2.0326e-02, -1.0452e-02, 4.1878e-03, -1.8927e-02, -4.9301e-02,\n -2.3229e-02, 1.7848e-03, 8.9783e-03, -1.2318e-02, 5.6052e-45,\n -1.9077e-02, 1.0073e-02, 1.1951e-02, -4.6252e-04, 5.6052e-45,\n -8.4038e-03, -4.9877e-03, 2.4088e-02, -1.3139e-02, -3.8403e-03,\n 2.5313e-03, -2.6408e-02, 7.3548e-04, -9.5045e-04, 3.6078e-03,\n 8.7422e-03, -1.8084e-02, -1.2749e-02, -1.7397e-04, -4.0832e-02,\n 1.0829e-02, -3.6196e-02, 6.3083e-23, 3.7562e-27, 3.7017e-03,\n 2.1969e-03, -1.5024e-02, -2.7693e-09, 2.8061e-03, -8.4934e-03,\n -1.6902e-02, 2.5158e-04, -9.0583e-03, -7.4322e-03, 2.2614e-02,\n 1.1432e-02, 7.6335e-03, 3.2598e-02, -8.4487e-03, 6.1142e-03,\n -4.9562e-03, 3.3930e-03, 2.8782e-02, -1.7597e-02, 8.4994e-04,\n 1.8249e-02, -2.3071e-02, 5.7733e-32, 8.4071e-30, -6.5391e-03,\n 2.4742e-02, -1.5463e-02, -3.7512e-02, 1.4492e-03, 1.0992e-03,\n -4.5201e-02, -1.0661e-02, 5.9124e-03, -4.8737e-04, 2.2978e-33,\n -2.1474e-05, -3.3565e-02, 1.7868e-02, -3.2502e-04, 1.2170e-05,\n 3.8928e-03, 9.7415e-03, 2.9308e-02, 1.1940e-02, 1.9191e-02,\n -8.1647e-03, 3.0442e-03, -3.8888e-03, -2.5077e-02, 1.2250e-02,\n -1.6413e-03, -2.5531e-02, -4.8607e-03, 2.5237e-02, 4.8066e-02,\n 7.1922e-03, -1.8733e-02, 1.1283e-02, 1.6394e-06, 4.4103e-03,\n 1.4819e-02, -8.5443e-03, -5.3989e-02, -1.8563e-02, -4.3936e-02,\n -1.3040e-02, 7.8469e-03, -6.3018e-02, 6.5663e-03, 6.0159e-03,\n 8.6966e-03, -7.3879e-03, -1.2783e-02, -3.4078e-02, 1.4064e-02,\n -2.5382e-02, -1.6579e-04, 2.1868e-02, -3.5772e-02, -6.2800e-03,\n 7.7235e-04, 1.8875e-03, -3.4342e-02, 2.7571e-02, -1.1460e-02,\n 3.1115e-02, -4.2305e-03, 7.3832e-40, -8.5955e-04, -1.5269e-02,\n 2.9632e-02, 5.7214e-02, 6.3015e-02, -3.4968e-02, 2.6324e-02,\n 1.2256e-02, 8.9374e-13, 3.2504e-02, -3.6663e-02, 2.8071e-19,\n 8.0190e-02, -5.5870e-03, -3.0679e-03, 2.4492e-03, 3.3198e-02,\n -2.3608e-03, 5.6052e-45, 1.9051e-02, 5.2085e-02, 1.8931e-02,\n 6.1182e-03, -6.7992e-02, 5.6052e-45, 5.1587e-03, 3.5890e-02,\n 2.1678e-02, -1.4406e-02, -5.4601e-03, 1.9144e-02, 1.2888e-02,\n 5.7364e-05, 1.1321e-02, -3.3445e-02, -1.4640e-04, 3.2436e-07,\n -2.9884e-03, -4.3738e-02, 1.6330e-17, -4.8169e-02, 9.6817e-03,\n 2.2606e-25, 3.1364e-02, 5.9806e-04, -2.9483e-02, -9.8904e-04,\n 2.6328e-03, 8.8098e-03], device='cuda:0')", - "exp_avg_sq": "tensor([1.8764e-02, 1.8272e-02, 2.1116e-06, 1.3778e-02, 3.6853e-03, 6.6573e-03,\n 1.8188e-02, 1.7011e-02, 1.8539e-02, 1.0614e-02, 3.9648e-03, 2.1009e-02,\n 3.1433e-03, 5.8576e-03, 1.6608e-02, 7.1616e-05, 4.8830e-03, 1.9907e-02,\n 6.7086e-03, 3.3539e-03, 4.3796e-03, 1.7794e-02, 5.7634e-03, 1.8685e-02,\n 1.5659e-02, 2.9941e-04, 1.5636e-02, 1.4540e-02, 1.8737e-02, 1.6553e-02,\n 1.5784e-02, 1.6078e-02, 1.6327e-02, 7.0099e-03, 1.0621e-02, 1.5467e-02,\n 1.3924e-03, 1.5506e-02, 1.8211e-02, 1.6032e-02, 1.1937e-02, 1.1553e-02,\n 4.5409e-04, 1.0012e-08, 1.4126e-02, 1.8995e-03, 2.2158e-02, 1.6567e-02,\n 1.8036e-02, 1.7208e-02, 4.1166e-04, 1.7573e-02, 3.8096e-03, 1.2772e-02,\n 2.0461e-02, 1.2595e-02, 2.1758e-02, 2.0982e-02, 1.5789e-02, 7.3397e-03,\n 1.5701e-02, 1.0106e-02, 1.5815e-02, 1.6533e-02, 2.0341e-02, 1.5778e-02,\n 5.0521e-03, 1.5900e-02, 2.2204e-02, 1.7102e-03, 1.6514e-02, 1.9365e-06,\n 8.2713e-03, 1.6110e-02, 1.5514e-02, 9.3026e-08, 1.3545e-02, 1.7353e-02,\n 1.5866e-02, 1.7169e-02, 1.9033e-02, 1.5958e-02, 8.2837e-03, 1.6548e-02,\n 2.2114e-02, 8.9850e-03, 1.6406e-02, 1.7232e-03, 1.2786e-02, 3.1307e-07,\n 4.9854e-03, 4.0865e-03, 3.4394e-07, 5.4478e-07, 1.4107e-02, 1.9926e-02,\n 1.0058e-04, 1.7903e-02, 1.6516e-02, 6.1304e-03, 4.3734e-03, 2.0879e-03,\n 3.7778e-03, 4.0758e-06, 1.6172e-02, 1.1247e-02, 1.8040e-02, 1.0636e-02,\n 1.0308e-02, 2.7883e-03, 5.5917e-06, 1.4616e-02, 6.2573e-03, 1.4898e-03,\n 1.7158e-02, 1.9321e-02, 1.9991e-02, 1.6465e-02, 8.9090e-03, 2.0490e-06,\n 1.1226e-07, 1.4761e-02, 8.8807e-03, 1.9667e-02, 1.7528e-02, 6.0540e-03,\n 1.6431e-02, 1.7108e-02, 1.7352e-02, 1.5799e-02, 1.6022e-02, 1.8787e-02,\n 3.5264e-03, 1.8629e-02, 1.8250e-02, 1.5643e-02, 1.5740e-02, 7.6894e-03,\n 1.5372e-02, 1.8558e-02, 2.0713e-02, 4.3730e-05, 2.0132e-02, 9.0030e-03,\n 1.6618e-02, 1.6743e-02, 1.6162e-02, 1.8333e-02, 1.7295e-02, 1.8789e-02,\n 1.4703e-02, 5.6712e-05, 1.4897e-02, 8.1449e-03, 2.0252e-02, 1.8296e-02,\n 5.1933e-08, 1.6048e-02, 9.2359e-04, 7.3904e-05, 1.8577e-02, 1.9959e-02,\n 1.3895e-02, 1.3569e-02, 1.6303e-02, 1.8068e-02, 1.5074e-02, 2.5697e-03,\n 1.5315e-02, 1.4183e-02, 1.5812e-02, 1.2953e-02, 6.2200e-03, 1.9376e-02,\n 1.8279e-02, 8.9495e-04, 1.8078e-02, 1.3957e-02, 1.7215e-02, 7.2711e-03,\n 1.5375e-02, 1.7679e-02, 3.1085e-06, 1.3844e-03, 1.1514e-02, 2.0096e-02,\n 1.6912e-02, 1.6843e-02, 5.8150e-03, 3.1123e-03, 6.3333e-03, 3.7299e-03,\n 1.5953e-02, 1.6468e-02, 2.5355e-04, 9.9471e-05, 1.1112e-03, 6.1182e-03,\n 1.0702e-02, 1.7909e-02, 1.3577e-03, 1.4747e-02, 9.0966e-03, 6.6425e-03,\n 1.2400e-02, 1.7522e-02, 2.6620e-03, 4.5561e-03, 4.0022e-03, 1.7254e-02,\n 6.1405e-06, 1.7645e-02, 1.8053e-02, 1.9364e-02, 2.1054e-02, 1.9905e-02,\n 1.6793e-02, 1.6094e-02, 1.1116e-02, 1.6410e-02, 1.6261e-02, 1.4181e-08,\n 9.1015e-03, 1.8053e-02, 5.7518e-03, 1.5482e-02, 2.0410e-04, 8.1102e-03,\n 1.8365e-02, 1.6995e-02, 1.0036e-03, 1.8236e-02, 1.5998e-02, 1.5762e-02,\n 1.6276e-02, 8.9184e-04, 1.7372e-02, 6.1681e-03, 1.8249e-02, 1.0992e-02,\n 1.4126e-02, 1.3468e-02, 1.8014e-02, 2.9758e-03, 1.7502e-02, 2.8353e-03,\n 1.8404e-02, 1.5680e-02, 1.7732e-02, 1.6440e-02, 1.8413e-02, 4.7115e-03,\n 1.8638e-02, 1.9537e-03, 1.8190e-02, 7.1188e-03, 8.9995e-03, 1.5712e-05,\n 1.6346e-02, 6.8524e-03, 6.3748e-07, 3.1674e-03, 9.1334e-07, 9.3132e-04,\n 5.6603e-03, 1.2174e-02, 1.5160e-02, 1.9009e-02, 1.5638e-02, 4.7673e-03,\n 1.7016e-02, 3.0314e-03, 1.9739e-02, 1.5944e-02, 1.2943e-02, 1.5510e-02,\n 8.6399e-03, 5.6596e-03, 6.2255e-03, 9.5688e-06, 1.6686e-02, 2.0533e-02,\n 1.8495e-02, 4.6201e-03, 4.1361e-03, 2.0182e-02, 3.7870e-03, 1.5436e-02,\n 1.1629e-02, 3.4890e-03, 1.0200e-02, 1.6730e-02, 1.2250e-02, 1.8554e-02,\n 6.7240e-03, 1.4706e-02, 1.7141e-02, 2.3400e-03, 2.9501e-06, 6.1673e-03,\n 7.6003e-03, 1.7310e-02, 1.4075e-02, 1.7002e-02, 1.7218e-02, 1.2266e-02,\n 1.5216e-02, 1.2400e-02, 1.8126e-02, 2.8434e-03, 1.3225e-02, 1.6580e-02,\n 1.9668e-02, 1.6248e-02, 1.6331e-02, 1.6534e-02, 2.1903e-03, 1.7074e-02,\n 2.6222e-03, 1.8165e-02, 1.4966e-02, 1.8902e-02, 1.3500e-02, 1.9297e-02,\n 7.4373e-03, 2.9604e-06, 1.3037e-05, 1.3654e-02, 1.4526e-02, 1.7412e-02,\n 1.7856e-02, 1.8401e-06, 1.7803e-02, 2.2227e-02, 1.4747e-04, 1.8303e-02,\n 3.3472e-03, 9.2779e-08, 1.2950e-06, 1.8184e-02, 1.4094e-02, 1.8324e-02,\n 1.6660e-02, 1.4634e-02, 1.3233e-02, 1.8590e-02, 1.6637e-02, 1.5577e-02,\n 1.6404e-02, 2.8314e-06, 1.8143e-03, 1.1277e-02, 1.6750e-02, 1.8065e-02,\n 2.6220e-06, 1.8991e-03, 1.8023e-02, 1.7743e-02, 1.2555e-02, 2.2086e-02,\n 1.7756e-02, 1.5363e-02, 1.6971e-02, 1.9232e-02, 1.6353e-02, 9.7520e-05,\n 1.6294e-02, 1.4263e-02, 1.6250e-02, 1.3664e-02, 3.4210e-03, 1.7195e-02,\n 2.6770e-06, 3.7029e-09, 2.2782e-03, 1.1965e-03, 5.5402e-03, 1.0826e-05,\n 1.7758e-02, 1.7853e-03, 1.6260e-02, 1.7396e-02, 8.8334e-03, 1.2946e-02,\n 4.7707e-03, 1.8341e-02, 1.5500e-02, 8.6413e-03, 1.4960e-02, 5.0860e-03,\n 1.8427e-02, 1.7727e-02, 1.8552e-02, 1.7232e-02, 1.5772e-02, 1.3550e-02,\n 1.9445e-02, 1.3188e-07, 1.7274e-05, 1.7283e-02, 1.4432e-02, 2.2199e-02,\n 1.2002e-02, 8.0722e-03, 2.4180e-03, 1.4398e-02, 1.2979e-02, 4.3121e-03,\n 7.7428e-03, 3.5046e-05, 1.9069e-05, 1.7086e-02, 1.0107e-02, 1.2835e-02,\n 3.0826e-05, 1.8859e-02, 1.6542e-02, 1.5748e-02, 2.0743e-03, 2.3803e-03,\n 1.6765e-02, 1.3605e-02, 5.1618e-03, 1.0833e-02, 1.4888e-02, 2.4556e-03,\n 1.4536e-02, 1.6998e-02, 9.2375e-03, 2.1265e-02, 2.0489e-03, 9.7723e-04,\n 1.7579e-02, 5.3052e-06, 1.5887e-04, 1.5638e-02, 1.5564e-02, 1.2871e-02,\n 1.6129e-02, 2.4327e-02, 4.8870e-03, 1.5943e-02, 2.0468e-02, 4.0398e-03,\n 1.9566e-02, 2.2624e-02, 1.2940e-02, 2.5075e-03, 1.6921e-02, 1.2306e-02,\n 1.5474e-02, 6.0880e-04, 1.8748e-02, 1.0882e-02, 1.6619e-03, 1.4346e-02,\n 1.6405e-02, 1.0783e-02, 1.7167e-02, 1.4428e-02, 1.5860e-02, 1.2360e-02,\n 1.4884e-09, 3.8985e-03, 6.8472e-03, 1.8204e-02, 1.3519e-02, 2.2676e-02,\n 1.2496e-02, 1.7735e-02, 1.2627e-02, 7.9700e-06, 1.8581e-02, 1.5395e-02,\n 1.4274e-05, 1.6384e-02, 7.2122e-03, 3.6154e-03, 1.5931e-02, 4.0571e-03,\n 1.8472e-02, 2.9771e-09, 2.8081e-03, 1.4730e-02, 1.5367e-02, 2.8912e-03,\n 2.0022e-02, 2.0257e-05, 1.3164e-02, 1.8537e-02, 1.3385e-02, 1.6409e-02,\n 1.2622e-02, 8.2167e-03, 1.4872e-02, 4.0220e-03, 1.2890e-02, 1.4525e-02,\n 5.0494e-04, 1.9957e-06, 4.1671e-04, 1.9199e-02, 3.1829e-07, 1.5725e-02,\n 1.4055e-02, 7.4773e-07, 1.3554e-02, 1.2199e-02, 5.8713e-03, 1.0020e-04,\n 1.9919e-04, 2.3390e-03], device='cuda:0')" + "step": "tensor(2504.)", + "exp_avg": "tensor([-3.8395e-02, -4.4321e-02, -1.0887e-07, 5.4965e-03, 2.5532e-02,\n 2.9340e-02, 8.3456e-04, -2.4231e-03, 1.0978e-02, -7.8966e-03,\n -2.7745e-03, -4.8707e-03, 1.6039e-03, 1.0304e-02, -1.2071e-02,\n 1.6101e-03, -6.5028e-03, 3.8275e-03, 2.0738e-02, 2.7832e-03,\n 1.1319e-02, 2.0668e-03, -2.1608e-02, 2.7022e-02, 5.5435e-04,\n -6.1991e-03, 4.6947e-03, -1.6725e-02, -1.0183e-02, 1.4969e-03,\n 6.4998e-03, -1.6628e-02, -2.0250e-02, 1.0783e-02, -4.0392e-02,\n -1.8706e-03, -6.1336e-03, 8.0045e-03, 2.0929e-03, 2.7848e-02,\n -7.6437e-04, 1.3769e-02, 3.7358e-03, 5.6052e-45, -8.6454e-03,\n -1.3804e-02, -2.3004e-02, 3.7751e-02, -1.6886e-02, 1.8700e-02,\n -1.2802e-02, -3.4758e-03, 3.6774e-03, 1.1410e-03, -1.5724e-02,\n 1.2025e-02, 1.8554e-02, 5.6089e-03, -7.5122e-03, 2.5646e-02,\n -5.0129e-02, -1.7277e-03, 8.3842e-03, 2.3831e-02, 1.7608e-02,\n 2.9526e-04, 4.6763e-03, -1.5616e-03, -1.7329e-02, 6.2349e-03,\n -9.9590e-03, -2.8709e-04, 1.1294e-03, -1.1719e-02, 2.6290e-02,\n 5.6052e-45, 2.4876e-02, -9.8164e-03, 9.6585e-03, -4.7360e-03,\n -1.5076e-02, 7.6075e-03, 7.0831e-03, -2.7819e-02, -1.9200e-05,\n -9.4828e-03, -2.5608e-02, 7.2438e-03, 1.7479e-02, 5.6052e-45,\n 6.1785e-03, 4.1592e-02, 4.8013e-03, 4.5864e-29, -1.7328e-02,\n 2.9284e-02, 1.0191e-03, -2.5349e-03, 3.0137e-02, 1.3797e-02,\n 6.3418e-03, -1.1831e-02, 4.6930e-03, 1.4616e-27, 1.1747e-04,\n 1.8723e-02, 7.9328e-03, -2.2623e-02, 2.8505e-03, 1.8274e-02,\n 5.6052e-45, -1.4578e-02, 1.0038e-02, 1.7921e-02, 2.4490e-02,\n -8.4617e-03, -2.0481e-02, -5.2342e-03, -5.4249e-03, -1.9461e-03,\n 1.1559e-25, 2.8088e-03, -8.9931e-03, 7.4840e-03, 1.4976e-02,\n -2.5514e-03, 3.5619e-02, -9.5267e-03, -4.9564e-03, -2.9207e-02,\n 5.3655e-02, -8.4511e-03, 1.0646e-02, 1.0108e-02, -1.1142e-02,\n -1.0037e-02, 1.6874e-02, 7.9351e-04, -2.2505e-02, -2.3943e-02,\n 3.3038e-02, 8.5853e-06, 2.2109e-02, -3.4412e-03, 4.0343e-03,\n 1.7346e-02, 2.1268e-02, -2.3974e-02, -3.3948e-02, 2.1941e-03,\n 2.5292e-02, -1.3725e-03, 2.8226e-03, 9.1456e-03, 2.7539e-02,\n -1.5420e-02, 5.6052e-45, 3.9145e-02, 6.6997e-04, 7.7687e-03,\n 1.0104e-02, -2.2469e-02, -5.8942e-03, 3.1084e-02, 1.8408e-02,\n 2.2230e-02, 1.9137e-02, 4.9120e-03, -1.3703e-03, 4.8362e-02,\n 1.2203e-02, -6.9771e-03, 1.5457e-02, 1.8743e-02, -4.9052e-02,\n 1.3612e-02, -1.7642e-02, 2.6352e-02, 2.9401e-03, 4.0362e-02,\n 1.0163e-02, 1.3184e-02, 3.0108e-26, 1.1971e-02, 5.7788e-03,\n 4.2549e-03, 3.2471e-02, -1.0222e-02, -2.0303e-02, -4.6226e-03,\n 1.7506e-02, 1.0158e-02, 1.6326e-02, 9.5873e-03, -1.3348e-04,\n 1.5934e-03, 5.4572e-03, -1.7504e-03, 2.5340e-02, -3.5331e-02,\n -2.3275e-02, -1.4643e-02, -2.0545e-02, -6.7234e-03, -1.3215e-03,\n -8.6201e-03, -7.6561e-03, 1.1084e-02, -4.3593e-03, -7.9776e-03,\n 2.5141e-06, -3.0306e-02, 1.4590e-03, 2.9013e-02, 2.1802e-02,\n -1.0265e-02, -1.1193e-02, -2.8215e-03, 4.1673e-03, 9.8934e-03,\n -2.6556e-02, 5.6052e-45, 1.2994e-02, -1.5281e-03, 1.5608e-02,\n -2.8651e-03, -5.6645e-03, 1.6759e-02, -2.7793e-02, -1.9376e-02,\n 5.3585e-03, -1.7172e-02, -2.1144e-03, -2.5872e-02, -2.8386e-02,\n -2.1399e-04, -3.5468e-03, -4.3035e-03, 1.0080e-02, -2.3247e-03,\n -2.1720e-03, 1.3021e-02, 3.8240e-03, -2.9180e-02, 1.5949e-02,\n 1.6719e-02, 1.7341e-02, 2.3670e-02, 9.3358e-03, -9.0272e-03,\n -2.9504e-03, -9.4805e-03, 2.0760e-02, 1.0553e-03, 8.8914e-03,\n 3.6088e-03, -7.4510e-03, -3.5429e-03, 2.5255e-02, -3.6365e-03,\n 5.6052e-45, -1.5212e-02, 5.6052e-45, 1.2104e-02, -2.2810e-04,\n 3.4950e-02, -1.1566e-03, -2.3107e-02, 1.4299e-02, 2.3136e-02,\n 1.7055e-02, -1.6583e-03, 8.2531e-03, -1.0191e-02, -2.2017e-02,\n -8.2280e-03, 7.5436e-03, 5.1537e-03, 6.2024e-03, 5.6515e-04,\n 8.8813e-03, 2.0657e-02, 1.3255e-03, 3.6300e-03, -2.1680e-03,\n -2.9756e-03, -8.2638e-03, 2.1847e-02, -5.6149e-03, -4.0028e-04,\n -1.8020e-02, 5.6393e-03, -1.6183e-02, -9.4929e-03, -2.0098e-02,\n -2.5405e-03, -1.7716e-02, 7.4426e-03, 5.0912e-23, -1.2416e-03,\n 6.0579e-03, 3.6110e-02, -1.0346e-02, 1.0519e-02, -1.1769e-02,\n -3.0567e-04, 3.3087e-02, 2.1046e-02, -1.6848e-03, -2.3133e-03,\n 4.5985e-03, 2.1854e-02, -1.1897e-02, 1.4604e-02, 2.9224e-02,\n -4.3310e-02, -3.5570e-04, -4.1367e-03, 6.9357e-03, -3.0390e-02,\n -2.2344e-03, -1.6272e-02, -5.4632e-03, -2.2777e-02, -1.1556e-03,\n 5.6052e-45, 5.6052e-45, 3.6517e-02, -7.0717e-03, -1.6227e-02,\n 4.3262e-03, 1.8970e-03, -3.4391e-02, -1.6473e-02, 4.5613e-05,\n -1.4565e-02, 3.4670e-04, 5.6052e-45, -1.4719e-03, 1.2029e-02,\n -1.6115e-03, 5.8186e-02, -7.2315e-03, -1.4288e-03, -1.6453e-02,\n -3.5657e-02, 9.3945e-03, -2.3533e-03, -1.1153e-02, 5.6052e-45,\n -2.3175e-03, -2.2951e-02, 2.5417e-02, -7.7497e-03, -1.3493e-14,\n 2.2456e-04, -4.0081e-03, 3.0983e-03, 1.3338e-02, 4.2651e-02,\n -2.6249e-02, -5.7835e-03, 3.1571e-02, -3.9528e-03, 1.9112e-02,\n 4.1580e-02, 1.1751e-02, -1.1318e-03, -2.7771e-04, -1.1534e-02,\n 5.6085e-03, -2.9821e-02, -9.5496e-04, 8.1424e-10, 6.2719e-03,\n -3.2340e-04, 5.5725e-03, -2.1753e-06, -5.9800e-03, 4.1290e-03,\n 4.7641e-02, 4.0755e-02, 9.5749e-03, -6.3976e-03, -2.2180e-02,\n -3.6415e-04, -4.6791e-03, 1.2564e-02, -2.1485e-02, -1.6796e-02,\n -1.6408e-02, 1.8620e-02, -3.5525e-02, 1.6668e-02, 1.2693e-02,\n 2.2266e-02, 2.2017e-02, 5.6052e-45, 3.0094e-03, 2.2907e-02,\n -9.1362e-03, -1.7878e-03, 8.3695e-03, -4.7427e-03, -3.8672e-03,\n -6.9100e-03, -2.1410e-02, -1.1738e-02, -9.7334e-03, 6.4139e-05,\n 3.2234e-03, 1.5659e-02, 2.0572e-02, 2.7073e-02, 1.4767e-02,\n 3.5939e-02, 2.0749e-02, 8.0664e-03, 3.2705e-03, 1.6759e-02,\n -2.7869e-02, -1.0269e-02, -8.5400e-03, 2.9622e-03, -8.5101e-03,\n -7.4980e-03, 1.2838e-02, -6.0411e-03, -2.7702e-03, 1.5708e-02,\n -7.1816e-03, -7.5576e-03, 5.5445e-03, 5.6052e-45, -1.5306e-02,\n -1.1775e-02, -2.2557e-03, 1.4048e-02, -1.8107e-02, -2.8384e-02,\n 5.3973e-03, 2.3993e-02, 3.3558e-02, 6.5597e-03, -2.1916e-02,\n -2.4847e-02, -9.7093e-03, 2.0541e-02, -8.9750e-03, 1.6115e-02,\n -8.8986e-03, -6.0879e-03, -1.2938e-02, 3.7077e-02, -1.5979e-02,\n -1.0842e-02, -8.1642e-03, 5.3801e-03, -3.0875e-04, 1.6411e-03,\n -1.3301e-02, 2.3577e-03, 2.3375e-04, -2.0513e-03, -2.3420e-02,\n 8.3886e-03, -1.1620e-02, 7.1503e-03, 1.2544e-02, -8.7852e-03,\n 1.7305e-02, 2.8282e-11, -3.3146e-02, 7.4974e-03, 1.1607e-03,\n 2.3214e-03, 6.6971e-03, -3.4464e-03, 3.5595e-03, -9.1093e-03,\n -1.0648e-03, 5.6052e-45, 1.0231e-02, 5.5386e-04, 1.2439e-02,\n 1.6877e-03, -6.2077e-03, 5.6052e-45, 9.6448e-04, -1.5821e-02,\n 1.7850e-03, 8.6314e-03, -4.2806e-03, 3.2053e-02, 4.0248e-03,\n -1.1998e-02, 1.2128e-02, 6.1182e-04, -5.6851e-03, 5.6052e-45,\n 1.8129e-02, 1.2307e-02, 5.6052e-45, 8.9654e-03, -2.3013e-02,\n 5.6052e-45, 3.1576e-02, 1.3341e-02, 1.3112e-02, -5.6545e-03,\n 4.9032e-03, -1.6296e-03], device='cuda:0')", + "exp_avg_sq": "tensor([1.0623e-02, 9.9821e-03, 6.0340e-07, 7.8364e-03, 2.8873e-03, 5.4046e-03,\n 8.9577e-03, 9.2026e-03, 9.8511e-03, 6.4240e-03, 2.4941e-03, 1.0851e-02,\n 2.2435e-03, 4.6133e-03, 9.2751e-03, 6.1225e-04, 3.5729e-03, 1.0917e-02,\n 4.1233e-03, 2.5166e-03, 3.7396e-03, 9.4149e-03, 4.1034e-03, 9.7841e-03,\n 8.5058e-03, 6.4166e-04, 9.0312e-03, 8.4075e-03, 1.0277e-02, 9.2985e-03,\n 8.2066e-03, 8.8215e-03, 8.6229e-03, 4.8644e-03, 6.7066e-03, 8.3229e-03,\n 1.5295e-03, 8.8736e-03, 9.2225e-03, 8.5839e-03, 6.8779e-03, 7.1613e-03,\n 1.2528e-03, 2.8611e-09, 7.9725e-03, 1.8944e-03, 1.1768e-02, 9.2755e-03,\n 9.9192e-03, 9.5842e-03, 1.1695e-03, 9.8899e-03, 2.9216e-03, 7.3367e-03,\n 1.0349e-02, 7.0300e-03, 1.1166e-02, 1.1992e-02, 9.3450e-03, 4.9832e-03,\n 8.9334e-03, 6.7671e-03, 8.9444e-03, 9.3106e-03, 1.1192e-02, 8.5015e-03,\n 4.5232e-03, 8.8743e-03, 1.1527e-02, 1.5828e-03, 9.1308e-03, 1.7684e-04,\n 5.2222e-03, 8.9337e-03, 8.2826e-03, 2.6583e-08, 7.8522e-03, 9.8908e-03,\n 9.2532e-03, 9.4409e-03, 9.5526e-03, 9.0750e-03, 5.7497e-03, 8.6886e-03,\n 1.1317e-02, 5.2050e-03, 8.8376e-03, 1.9285e-03, 7.4464e-03, 8.9463e-08,\n 3.5985e-03, 4.1849e-03, 2.9807e-04, 1.5590e-07, 8.9665e-03, 1.0965e-02,\n 5.0246e-04, 9.8520e-03, 8.9493e-03, 4.7727e-03, 3.6445e-03, 2.1885e-03,\n 3.1492e-03, 1.5901e-06, 8.4958e-03, 7.0041e-03, 1.0107e-02, 6.3509e-03,\n 6.3430e-03, 2.5292e-03, 1.5979e-06, 7.8230e-03, 4.0203e-03, 2.2110e-03,\n 1.0087e-02, 1.0560e-02, 1.0603e-02, 9.6675e-03, 5.7332e-03, 2.0026e-04,\n 2.6807e-05, 8.8622e-03, 6.2086e-03, 1.0409e-02, 9.5797e-03, 4.1717e-03,\n 9.2717e-03, 9.4539e-03, 9.4124e-03, 8.6823e-03, 9.3491e-03, 1.0548e-02,\n 2.4697e-03, 1.0109e-02, 9.2956e-03, 8.6464e-03, 8.7556e-03, 5.0566e-03,\n 8.6096e-03, 1.0180e-02, 1.0700e-02, 1.2500e-05, 1.1130e-02, 5.7462e-03,\n 8.9139e-03, 9.4750e-03, 8.5206e-03, 9.5086e-03, 9.4260e-03, 1.0006e-02,\n 8.0384e-03, 3.7036e-04, 8.4888e-03, 5.5911e-03, 1.0021e-02, 9.7023e-03,\n 1.4840e-08, 9.0239e-03, 1.2117e-03, 5.5115e-04, 1.0109e-02, 1.0575e-02,\n 8.2120e-03, 7.3738e-03, 8.8718e-03, 9.4526e-03, 8.6423e-03, 2.0466e-03,\n 8.4322e-03, 7.6863e-03, 8.9523e-03, 7.3989e-03, 4.5860e-03, 1.0606e-02,\n 9.5849e-03, 8.9906e-04, 9.5420e-03, 8.0833e-03, 9.3710e-03, 5.1000e-03,\n 8.5123e-03, 9.3717e-03, 2.8417e-06, 1.9461e-03, 7.2129e-03, 1.0230e-02,\n 9.1399e-03, 8.8697e-03, 4.3809e-03, 3.0958e-03, 5.4714e-03, 2.7364e-03,\n 9.0135e-03, 9.6733e-03, 7.8931e-04, 5.6903e-04, 1.0089e-03, 4.0908e-03,\n 6.3167e-03, 9.5319e-03, 1.7458e-03, 8.1976e-03, 5.5008e-03, 4.3986e-03,\n 7.4954e-03, 9.1531e-03, 2.3267e-03, 3.9345e-03, 2.9105e-03, 1.0027e-02,\n 1.7588e-06, 9.2946e-03, 9.8480e-03, 1.0300e-02, 1.0997e-02, 1.1179e-02,\n 9.7033e-03, 9.0179e-03, 6.3947e-03, 9.3479e-03, 9.5081e-03, 4.0558e-09,\n 5.7027e-03, 9.5935e-03, 4.1189e-03, 8.6936e-03, 8.4731e-04, 5.4602e-03,\n 1.0035e-02, 8.7759e-03, 1.1500e-03, 9.4753e-03, 9.5903e-03, 8.8855e-03,\n 9.3215e-03, 1.6955e-03, 9.2453e-03, 4.2682e-03, 9.9808e-03, 6.2731e-03,\n 8.0215e-03, 7.7157e-03, 9.8098e-03, 2.4622e-03, 9.8461e-03, 2.5088e-03,\n 9.3694e-03, 9.0035e-03, 1.0277e-02, 9.0354e-03, 1.0111e-02, 3.4592e-03,\n 9.9233e-03, 1.9983e-03, 9.8236e-03, 5.7900e-03, 6.2010e-03, 4.4713e-05,\n 9.3892e-03, 5.1311e-03, 1.8217e-07, 2.0659e-03, 2.6100e-07, 1.2303e-03,\n 4.4144e-03, 7.2117e-03, 8.7763e-03, 1.0471e-02, 8.9552e-03, 4.1405e-03,\n 9.7579e-03, 2.3994e-03, 1.0056e-02, 8.9133e-03, 7.9449e-03, 8.1109e-03,\n 5.5320e-03, 4.0756e-03, 3.9827e-03, 3.9095e-05, 9.5362e-03, 1.1094e-02,\n 9.8212e-03, 3.2239e-03, 2.9488e-03, 1.0519e-02, 3.0211e-03, 9.5208e-03,\n 6.5749e-03, 3.0815e-03, 5.9677e-03, 9.8964e-03, 7.4756e-03, 9.7594e-03,\n 4.4479e-03, 8.1591e-03, 9.1807e-03, 1.9118e-03, 5.5313e-06, 3.9736e-03,\n 4.8685e-03, 9.6478e-03, 8.5437e-03, 9.3591e-03, 9.8366e-03, 6.6840e-03,\n 8.5422e-03, 7.5437e-03, 9.8521e-03, 2.3122e-03, 7.8178e-03, 8.8897e-03,\n 1.0459e-02, 8.7864e-03, 9.4340e-03, 8.9552e-03, 1.8345e-03, 9.2248e-03,\n 1.8005e-03, 1.0244e-02, 8.1561e-03, 1.0322e-02, 8.0229e-03, 1.0349e-02,\n 5.3447e-03, 8.4595e-07, 3.7255e-06, 7.6133e-03, 8.0742e-03, 9.6520e-03,\n 9.3626e-03, 2.6846e-05, 9.2160e-03, 1.1805e-02, 3.9818e-04, 9.5105e-03,\n 2.3305e-03, 2.6512e-08, 1.3443e-05, 1.0203e-02, 8.3244e-03, 1.0653e-02,\n 9.1463e-03, 8.2808e-03, 7.5812e-03, 1.0201e-02, 9.1278e-03, 9.1740e-03,\n 8.4352e-03, 8.0909e-07, 1.8125e-03, 6.8162e-03, 9.0836e-03, 1.1064e-02,\n 7.5539e-07, 1.9746e-03, 9.4993e-03, 9.4075e-03, 7.5357e-03, 1.1446e-02,\n 9.8374e-03, 8.2622e-03, 9.3643e-03, 1.0323e-02, 9.0314e-03, 1.0533e-03,\n 9.2557e-03, 8.2116e-03, 9.3805e-03, 7.5551e-03, 2.6372e-03, 8.9316e-03,\n 1.2729e-04, 1.0582e-09, 1.6439e-03, 1.1368e-03, 4.2909e-03, 6.1762e-06,\n 9.0852e-03, 1.6226e-03, 9.3680e-03, 9.4410e-03, 5.6297e-03, 7.1443e-03,\n 3.7231e-03, 9.4999e-03, 9.2733e-03, 5.5080e-03, 8.2818e-03, 3.3503e-03,\n 1.0085e-02, 1.0083e-02, 1.0139e-02, 9.6047e-03, 9.1517e-03, 7.6948e-03,\n 1.0344e-02, 3.7696e-08, 1.7505e-04, 9.7913e-03, 8.5634e-03, 1.1651e-02,\n 7.5088e-03, 5.0656e-03, 2.3628e-03, 8.7791e-03, 7.3774e-03, 3.0906e-03,\n 5.3268e-03, 1.0026e-05, 2.9315e-04, 8.9531e-03, 6.6699e-03, 7.6258e-03,\n 2.9112e-04, 1.0224e-02, 9.3999e-03, 9.1398e-03, 2.2666e-03, 2.1632e-03,\n 9.0840e-03, 7.8473e-03, 4.2578e-03, 6.8323e-03, 8.6399e-03, 1.9567e-03,\n 8.0847e-03, 9.4321e-03, 5.7783e-03, 1.1098e-02, 2.5060e-03, 1.7933e-03,\n 9.6175e-03, 1.5160e-06, 7.3468e-04, 8.2134e-03, 8.7738e-03, 7.3979e-03,\n 8.8112e-03, 1.2034e-02, 3.4416e-03, 9.1833e-03, 1.0658e-02, 2.8299e-03,\n 9.9784e-03, 1.2342e-02, 7.6607e-03, 2.4543e-03, 9.2223e-03, 7.1975e-03,\n 8.8286e-03, 1.2936e-03, 9.5053e-03, 6.9102e-03, 1.9586e-03, 8.4345e-03,\n 9.4793e-03, 6.4795e-03, 8.9881e-03, 8.0305e-03, 9.2204e-03, 7.6057e-03,\n 1.6346e-04, 3.0100e-03, 4.1954e-03, 1.0041e-02, 7.3329e-03, 1.2556e-02,\n 7.2214e-03, 9.9157e-03, 7.7393e-03, 2.3270e-06, 1.0057e-02, 8.5338e-03,\n 1.7566e-05, 9.1632e-03, 5.2430e-03, 2.8192e-03, 8.9242e-03, 3.4241e-03,\n 9.8553e-03, 8.5074e-10, 2.2134e-03, 7.9635e-03, 8.5916e-03, 2.3859e-03,\n 1.0573e-02, 5.7894e-06, 7.5675e-03, 9.9563e-03, 7.3315e-03, 8.7530e-03,\n 7.6910e-03, 6.0972e-03, 8.1482e-03, 3.0951e-03, 7.3905e-03, 8.8521e-03,\n 1.1322e-03, 5.7028e-07, 1.0319e-03, 9.8199e-03, 9.0954e-08, 8.3712e-03,\n 7.7379e-03, 6.1651e-07, 7.8780e-03, 6.7962e-03, 4.3014e-03, 3.7004e-04,\n 6.3751e-04, 1.6462e-03], device='cuda:0')" }, "4": { + "step": "tensor(2504.)", + "exp_avg": "tensor([[ 2.2412e-04, 4.0157e-04, 1.0679e-10, ..., 8.8400e-05,\n 2.9088e-05, 1.1721e-05],\n [-9.6827e-04, -3.1248e-04, 5.0244e-10, ..., -1.0414e-06,\n 3.1966e-05, -1.0408e-05],\n [ 2.5750e-04, 9.0565e-04, -5.9843e-10, ..., 1.4278e-05,\n -7.9051e-05, 5.1686e-05],\n ...,\n [ 3.4903e-05, -4.6048e-04, 1.0136e-09, ..., -4.9507e-05,\n 2.5272e-05, 6.9145e-05],\n [ 1.2258e-04, 7.3625e-04, -3.8014e-10, ..., 6.5449e-05,\n 3.1657e-05, -2.5590e-05],\n [-2.0758e-04, -1.1146e-03, -1.1291e-09, ..., 2.4395e-05,\n 8.3260e-05, 8.1837e-05]], device='cuda:0')", + "exp_avg_sq": "tensor([[8.3057e-07, 9.2900e-07, 3.1569e-11, ..., 4.8782e-08, 4.2299e-08,\n 1.1593e-08],\n [1.5006e-06, 1.8763e-06, 4.8125e-11, ..., 1.0404e-08, 7.6373e-08,\n 6.3056e-08],\n [1.3558e-06, 1.7666e-06, 1.3309e-10, ..., 2.2539e-08, 2.9669e-08,\n 4.8070e-08],\n ...,\n [1.3393e-06, 1.9916e-06, 4.4261e-11, ..., 2.0266e-08, 5.3962e-08,\n 4.1967e-08],\n [1.2168e-06, 1.9979e-06, 3.1024e-11, ..., 2.5704e-08, 4.6682e-08,\n 3.9930e-08],\n [1.4679e-06, 2.3003e-06, 4.7319e-11, ..., 1.8116e-08, 6.5686e-08,\n 2.4935e-08]], device='cuda:0')" + }, + "5": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-1.9408e-06, 2.4689e-05, -6.1540e-08, ..., 5.1586e-06,\n 1.4104e-05, -6.4331e-06],\n [-4.7889e-06, 1.0331e-05, 1.5720e-06, ..., 2.4125e-05,\n 1.6149e-06, -2.6210e-05],\n [ 1.8135e-05, 5.5189e-06, -7.9151e-06, ..., -3.7316e-07,\n 6.3259e-06, 5.8550e-06],\n ...,\n [ 4.0401e-06, 6.7720e-07, -2.2175e-06, ..., 9.5534e-06,\n -2.5897e-06, -2.9794e-06],\n [-1.9786e-06, -1.6281e-05, -5.7256e-06, ..., -2.6579e-06,\n -2.5889e-06, -3.8625e-05],\n [ 2.4746e-07, 4.2050e-06, 8.0501e-07, ..., 3.0689e-06,\n 2.6812e-05, 4.9440e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.0791e-09, 2.6889e-09, 8.9253e-10, ..., 1.0173e-09, 1.3098e-09,\n 1.2916e-09],\n [1.0009e-09, 1.9336e-09, 5.2529e-10, ..., 6.1637e-09, 6.9941e-10,\n 1.1058e-09],\n [3.7998e-09, 1.0052e-09, 1.3661e-09, ..., 6.2990e-10, 1.0629e-09,\n 1.3711e-09],\n ...,\n [3.1857e-09, 1.3944e-09, 1.0315e-09, ..., 1.9080e-09, 1.3613e-09,\n 8.9508e-10],\n [1.3169e-09, 3.8868e-09, 1.3366e-09, ..., 2.9471e-09, 1.5748e-09,\n 1.7645e-09],\n [5.0013e-09, 6.0751e-10, 8.1609e-10, ..., 1.2286e-09, 2.1974e-09,\n 1.0453e-09]], device='cuda:0')" + }, + "6": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-8.9299e-05, -4.0434e-04, 1.9845e-03, ..., -2.4182e-03,\n 1.7099e-04, 8.9060e-04], device='cuda:0')", + "exp_avg_sq": "tensor([2.8872e-05, 2.2807e-05, 3.2958e-05, ..., 2.4312e-05, 3.1933e-05,\n 2.8954e-05], device='cuda:0')" + }, + "7": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.1164e-05, -2.3875e-07, -2.8098e-06, ..., -2.7049e-06,\n -2.2046e-05, -1.5502e-05],\n [-1.2148e-05, -7.7866e-06, -1.2685e-05, ..., -2.3404e-05,\n -1.1515e-05, 4.9558e-06],\n [-2.2966e-05, 2.7019e-05, -6.5322e-06, ..., 1.6928e-05,\n -5.5988e-06, 1.5700e-05],\n ...,\n [-1.6920e-06, -4.3450e-06, 5.4849e-05, ..., 7.1327e-06,\n 1.9451e-05, 9.7505e-06],\n [ 8.2346e-06, 1.1570e-05, -9.2320e-06, ..., 2.4148e-06,\n 2.3644e-06, -1.3245e-05],\n [-3.3576e-06, 1.1844e-05, 2.0205e-06, ..., -7.9653e-06,\n -5.4172e-06, 3.8898e-06]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.3694e-09, 6.3742e-10, 1.5943e-09, ..., 1.0463e-09, 2.3084e-09,\n 1.4791e-09],\n [2.3304e-09, 1.5558e-09, 2.5566e-09, ..., 2.3402e-09, 3.5760e-09,\n 2.4302e-09],\n [3.7682e-09, 1.7676e-09, 4.3534e-09, ..., 2.6358e-09, 4.5959e-09,\n 2.9130e-09],\n ...,\n [2.9436e-09, 1.9991e-09, 3.5103e-09, ..., 2.7451e-09, 4.1252e-09,\n 2.8258e-09],\n [2.5204e-09, 1.6429e-09, 2.5395e-09, ..., 2.4375e-09, 3.2399e-09,\n 2.2250e-09],\n [2.8182e-09, 1.6000e-09, 3.2927e-09, ..., 2.3790e-09, 3.3453e-09,\n 3.0617e-09]], device='cuda:0')" + }, + "32": { + "step": "tensor(1252.)", + "exp_avg": "tensor([3.4150e-14], device='cuda:0')", + "exp_avg_sq": "tensor([0.0023], device='cuda:0')" + }, + "33": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 3.7158e-16, -3.5958e-17, -3.3562e-16], device='cuda:0')", + "exp_avg_sq": "tensor([6.5965e-07, 1.1026e-06, 1.0298e-07], device='cuda:0')" + }, + "34": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.6052e-45, -1.7651e-16, 2.5765e-16, 1.3458e-16, 1.4535e-16,\n 2.4838e-16, 2.9569e-16, 2.8077e-16, 1.1523e-15, 3.3731e-16],\n device='cuda:0')", + "exp_avg_sq": "tensor([8.7064e-04, 1.0460e-05, 1.0063e-05, 7.9187e-06, 9.5143e-06, 8.8292e-06,\n 1.4802e-05, 1.3237e-05, 1.4296e-05, 9.3096e-06], device='cuda:0')" + }, + "36": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 1.7138e-18, 4.5492e-19, 8.7810e-21, ..., 6.1922e-19,\n 3.4819e-19, 4.5603e-19],\n [ 3.6387e-19, -1.9457e-20, -2.5627e-19, ..., 2.4165e-19,\n -2.3434e-19, -3.4303e-20],\n [ 8.3566e-19, 7.5378e-20, 1.5826e-19, ..., 2.8456e-19,\n 3.8837e-19, 5.4436e-20],\n ...,\n [ 2.1214e-19, 2.8088e-19, -5.2753e-20, ..., 4.9975e-19,\n 2.2725e-19, 1.8274e-19],\n [-3.6354e-18, -1.4800e-18, 6.2102e-19, ..., -3.2005e-18,\n -8.9060e-19, -8.2585e-19],\n [ 4.4010e-19, 1.2446e-19, -6.2888e-19, ..., -5.4075e-19,\n 8.6776e-20, 2.7001e-19]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.1885e-10, 2.6193e-11, 7.0782e-11, ..., 8.1330e-11, 1.0257e-10,\n 1.3841e-10],\n [3.2847e-13, 5.8926e-13, 8.3244e-13, ..., 4.7812e-13, 4.0735e-13,\n 7.9067e-13],\n [2.4009e-12, 1.2674e-12, 3.1058e-12, ..., 1.3291e-12, 4.8908e-12,\n 2.5767e-12],\n ...,\n [2.8133e-12, 2.0140e-12, 1.8711e-12, ..., 3.1871e-12, 2.1526e-12,\n 4.6115e-12],\n [6.1153e-11, 1.5642e-11, 4.8567e-11, ..., 3.2477e-11, 6.1111e-11,\n 6.6524e-11],\n [4.2131e-11, 2.2628e-11, 5.1167e-11, ..., 1.9229e-11, 5.3886e-11,\n 6.7734e-11]], device='cuda:0')" + }, + "37": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 3.2179e-16, -3.1781e-17, 1.3202e-16, -4.6249e-17, -9.0206e-18,\n 5.7575e-16, 2.1262e-17, -9.7131e-17, -9.1563e-16, -1.0400e-15,\n -1.1020e-15, 6.6161e-16, -4.9164e-16, 5.7264e-16, 2.1081e-16,\n 5.3173e-16, -8.9229e-16, 6.4850e-16, 7.6626e-16, -1.5536e-17,\n -9.5735e-17, 1.2534e-16, 7.0091e-17, 8.8209e-16, 2.2705e-17,\n -1.5242e-15, 1.8084e-16, -1.5554e-15, -7.3297e-16, -7.0411e-18,\n -2.0215e-16, -1.4788e-15, 4.4399e-16, 1.4263e-15, -3.4735e-16,\n -1.2019e-16, 9.4914e-17, 3.8142e-16, 4.3003e-18, 6.2299e-16,\n 8.0946e-17, -4.6900e-16, 4.8176e-16, 4.9180e-16, -7.2501e-16,\n 3.1768e-17, 9.1963e-16, -1.1485e-15, 5.5442e-16, 5.8843e-16,\n 9.6077e-16, 4.2445e-17, -1.5614e-16, 3.0580e-16, 2.5581e-16,\n 2.2778e-17, 3.4247e-16, -2.9392e-16, -2.3783e-16, -2.9578e-16,\n 2.8070e-16, 6.7271e-17, -1.8579e-15, -1.8606e-15, -4.7221e-16,\n 2.0574e-16, 8.0947e-17, 3.0170e-16, -3.3850e-16, 3.5031e-16,\n -4.0037e-16, 8.3891e-16, 1.1638e-17, -6.1788e-17, 2.5441e-16,\n 1.5698e-16, -1.3553e-16, -5.5873e-16, 2.2648e-16, -8.3613e-17,\n -2.5824e-16, -6.6331e-16, -4.2035e-17, -3.2365e-17, -8.3691e-16,\n 9.9778e-17, -6.5884e-17, 1.1601e-15, -1.7360e-16, -6.1951e-16,\n 9.1431e-16, 4.4278e-16, -9.0051e-17, 1.3743e-16, -2.8447e-17,\n -5.1171e-17, 6.5113e-16, -1.3548e-15, -5.8787e-17, 1.5056e-16,\n 5.8236e-16, 7.8080e-16, 1.1982e-15, 1.6297e-16, 4.2373e-16,\n 1.1826e-15, -1.1600e-15, -3.0346e-17, 9.2132e-17, 7.9434e-16,\n 4.3938e-16, 3.4389e-16, -2.8611e-16, 8.7786e-16, -1.3482e-17,\n -2.3671e-16, -6.9822e-16, 3.3526e-16, 1.8884e-16, 2.4255e-16,\n -5.0152e-16, 1.5751e-16, -2.1094e-16, -7.5867e-16, 5.7707e-16,\n 1.4217e-16, -3.1732e-16, 6.4177e-16, -1.3112e-16, 4.5922e-16,\n 4.2199e-16, -1.8466e-16, -3.8866e-16, 2.9656e-16, 9.9854e-17,\n -1.7210e-15, 6.6025e-17, -7.4915e-17, -1.4142e-17, 6.2285e-16,\n 6.1117e-17, -5.0160e-16, -6.0641e-16, 3.3945e-18, -2.2670e-17,\n -4.2641e-17, -2.5985e-16, -8.1516e-17, 1.7023e-16, 6.6827e-16,\n 5.0288e-17, -1.3980e-15, -4.2793e-16, 1.5140e-15, -1.2440e-16,\n 7.1387e-17, -1.6616e-15, 4.1438e-16, -4.1221e-16, -8.9493e-16,\n -5.9678e-16, -7.5867e-17, -8.9034e-18, -4.9969e-16, 1.0207e-15,\n 3.7642e-16, -1.8874e-16, 5.2205e-16, 1.1256e-15, 1.3200e-16,\n -7.9375e-16, 5.9958e-17, 8.3792e-16, 6.8898e-16, 7.0514e-17,\n -3.3183e-16, 2.6850e-16, 2.9736e-16, 9.2656e-16, 1.0174e-15,\n 5.2467e-16, 1.4296e-16, 4.6687e-16, 3.5870e-17, -3.6120e-16,\n -5.4909e-16, -9.4819e-17, 2.8638e-17, 3.8537e-16, 7.6379e-18,\n -2.5969e-16, 1.0496e-16, -1.8326e-16, 8.7827e-17, 1.5712e-16,\n 1.0967e-16, 3.3268e-17, 2.8167e-16, -1.7919e-15, 6.8126e-16,\n 2.2754e-16, -1.2286e-16, 3.9865e-16, -1.5992e-15, -6.2505e-16,\n 5.8790e-16, -9.8831e-16, 1.3005e-16, -1.5637e-17, -6.1661e-16,\n 1.2123e-15, 1.8127e-16, -1.4481e-16, 9.3789e-17, 2.8291e-16,\n 1.8544e-15, 7.5659e-16, -4.6335e-16, 2.5526e-16, -1.1561e-16,\n 3.5848e-16, -2.7021e-16, 5.4263e-17, -1.1283e-15, -4.6951e-16,\n -8.5656e-16, 1.4400e-17, 1.8342e-16, -1.3710e-16, 3.4577e-16,\n 5.4091e-16, 4.6525e-16, -3.8947e-16, -9.7331e-16, -3.1847e-16,\n 5.1393e-16, 6.5587e-16, 3.9850e-16, -6.6002e-16, -1.7832e-16,\n 5.0939e-17, 3.2592e-16, -8.8974e-16, 6.5496e-16, 8.3435e-16,\n 4.4052e-16, -1.4113e-15, 3.8272e-16, -8.1762e-16, -7.3744e-17,\n -5.4103e-17, 6.4523e-17, 5.0600e-17, 1.4548e-16, -9.5111e-16,\n 1.9943e-16], device='cuda:0')", + "exp_avg_sq": "tensor([2.2763e-05, 2.1964e-07, 5.7059e-07, 4.4530e-06, 1.0131e-08, 8.7395e-06,\n 6.3501e-08, 2.4093e-06, 2.3966e-07, 8.5061e-06, 2.0693e-06, 3.6170e-05,\n 1.7983e-05, 9.6455e-07, 1.6482e-06, 4.9842e-06, 8.7330e-07, 1.0635e-06,\n 7.2161e-06, 1.4702e-07, 6.0383e-09, 3.3670e-09, 1.2764e-06, 3.6404e-06,\n 3.5314e-09, 9.3201e-05, 1.7421e-06, 5.6938e-05, 1.8400e-06, 4.5445e-06,\n 1.3679e-06, 1.4923e-05, 3.0014e-06, 9.6608e-06, 2.4834e-06, 4.9193e-06,\n 9.7129e-07, 5.2929e-05, 5.8818e-09, 5.8756e-07, 6.6663e-09, 1.4362e-07,\n 3.1600e-08, 5.0227e-09, 5.0739e-06, 4.0881e-05, 2.7084e-07, 3.9056e-05,\n 3.6632e-05, 4.7452e-07, 9.1769e-06, 4.2900e-06, 1.0219e-07, 9.3598e-07,\n 2.3123e-06, 4.9362e-06, 4.5622e-08, 2.2091e-07, 6.2695e-06, 4.1223e-06,\n 6.8045e-06, 3.8698e-08, 2.9236e-05, 3.1986e-05, 2.6610e-08, 5.2573e-08,\n 5.5963e-07, 2.1753e-05, 4.5521e-07, 1.0022e-05, 1.9518e-07, 5.9014e-07,\n 4.5666e-07, 2.3611e-06, 2.9128e-05, 1.0169e-09, 3.8947e-08, 9.1533e-07,\n 9.7135e-06, 7.9381e-07, 2.4976e-07, 1.4125e-07, 1.8403e-05, 1.7115e-08,\n 4.3104e-07, 1.2005e-06, 5.9575e-06, 2.9411e-05, 5.7331e-06, 2.6791e-07,\n 4.1519e-05, 9.0131e-06, 2.9570e-08, 1.7202e-06, 1.5961e-07, 2.2754e-07,\n 3.7384e-05, 3.9760e-06, 3.0664e-07, 7.5113e-08, 6.2483e-06, 2.4296e-06,\n 3.2728e-05, 6.8893e-07, 2.7926e-05, 3.0089e-05, 3.0944e-08, 3.6107e-06,\n 3.9303e-07, 6.4577e-07, 7.7030e-06, 3.7195e-09, 5.6335e-08, 7.9756e-06,\n 3.2891e-08, 7.2918e-06, 1.9956e-05, 2.8900e-08, 1.8717e-06, 9.7545e-07,\n 6.2235e-06, 9.9948e-07, 9.5125e-08, 2.2021e-07, 4.5800e-06, 1.8407e-05,\n 3.6692e-06, 2.4206e-05, 5.8428e-06, 6.9118e-05, 1.0682e-06, 1.1680e-07,\n 9.3292e-07, 1.5307e-05, 2.6479e-09, 1.3644e-05, 3.8849e-07, 4.1817e-09,\n 4.1859e-06, 5.7994e-06, 1.0188e-05, 3.3684e-05, 1.0140e-07, 6.2911e-08,\n 1.3951e-08, 3.1763e-07, 2.2473e-05, 1.6072e-06, 2.0664e-06, 2.8525e-06,\n 7.6686e-09, 1.0166e-04, 2.3542e-07, 7.4287e-07, 4.2007e-09, 1.7461e-05,\n 7.8875e-05, 3.9905e-07, 5.1599e-07, 6.4864e-06, 1.1965e-05, 3.2759e-08,\n 4.4518e-09, 3.5666e-05, 8.9172e-06, 1.0388e-06, 3.5418e-07, 1.6156e-05,\n 9.0025e-06, 8.2079e-06, 1.7848e-05, 6.2085e-06, 3.4887e-05, 2.9168e-07,\n 2.0926e-06, 9.7160e-07, 3.1377e-07, 9.3426e-06, 7.4415e-07, 9.6433e-06,\n 4.4021e-05, 1.2237e-08, 1.1488e-05, 3.2882e-06, 2.8289e-06, 2.0863e-05,\n 2.7550e-06, 1.4721e-06, 2.3286e-06, 2.1387e-05, 7.3922e-07, 1.4686e-06,\n 2.1349e-06, 2.3265e-05, 1.6292e-05, 1.7162e-07, 5.7143e-08, 4.0866e-06,\n 2.3612e-06, 5.3946e-06, 2.0008e-06, 1.2213e-04, 2.0032e-06, 1.7354e-05,\n 8.5371e-06, 2.6980e-05, 1.3710e-05, 9.4673e-09, 8.2686e-06, 1.6079e-05,\n 5.1622e-05, 1.4123e-08, 9.6327e-07, 2.5686e-08, 3.9163e-07, 8.3928e-06,\n 1.5379e-06, 7.4051e-08, 1.3245e-08, 3.4606e-08, 2.3584e-05, 1.2915e-05,\n 3.3202e-06, 3.5243e-05, 4.0071e-06, 1.4616e-05, 1.6133e-07, 4.1880e-06,\n 8.4984e-07, 3.4168e-06, 3.4378e-05, 1.7927e-05, 4.2139e-06, 5.5665e-06,\n 8.1534e-07, 1.7710e-06, 2.2287e-07, 7.7765e-05, 2.1418e-06, 3.1637e-06,\n 4.0703e-06, 5.1253e-05, 3.8233e-06, 2.1801e-07, 4.9391e-07, 1.1282e-07,\n 5.1175e-06, 2.2536e-07, 2.3327e-07, 1.7302e-06, 7.8236e-07, 1.7900e-05,\n 2.9425e-07, 9.3580e-07, 1.1720e-05, 1.3295e-05], device='cuda:0')" + }, + "38": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-2.5515e-18, -6.1723e-18, -1.9898e-19, -1.1843e-16, -1.0503e-18,\n -1.4848e-19, -4.0425e-17, -5.2541e-19, -1.1659e-17, -1.5657e-16,\n -4.0375e-17, -1.0457e-19, -1.6281e-16, -2.1001e-17, 3.7274e-19,\n -7.4368e-17, -1.8698e-17, -3.0272e-19, -1.5461e-18, 3.5598e-19,\n -2.1297e-18, 4.2022e-20, 2.4760e-19, -4.6342e-17, -1.1059e-18,\n -2.2753e-16, 8.3117e-19, -1.6853e-16, -1.2439e-16, -7.0332e-19,\n -3.1339e-18, -1.1787e-16, -1.2342e-17, 2.0865e-19, 2.6051e-18,\n -5.1969e-19, 6.0908e-19, 2.4109e-19, -7.3381e-19, 9.4315e-20,\n 1.3408e-18, -2.7455e-17, -8.6865e-19, -9.7663e-19, -1.0827e-17,\n 2.2192e-18, -7.7896e-17, -1.7560e-16, -3.0633e-18, -3.9582e-17,\n -7.3555e-17, -8.9836e-21, -1.0710e-17, -2.7214e-19, -5.2679e-17,\n -1.5251e-17, 3.5902e-19, -2.8664e-18, -6.3461e-17, -1.6404e-16,\n -6.9032e-20, 2.4974e-19, -9.4317e-17, -1.7196e-16, -2.4814e-18,\n 2.6510e-19, 4.7209e-19, -1.7580e-19, -2.3370e-17, -4.1292e-19,\n -1.2270e-17, -9.0659e-19, -5.1114e-19, -1.5253e-18, -1.4259e-16,\n 5.8851e-19, -3.7075e-18, 4.7171e-18, -2.8837e-19, -4.6623e-18,\n -5.6095e-18, -1.0033e-17, -2.0135e-18, -1.6967e-18, -4.1600e-17,\n 8.3480e-19, -1.5568e-17, -9.5529e-20, 3.3129e-18, -4.3341e-17,\n -2.9622e-17, 8.9502e-20, -5.9358e-18, -6.3018e-19, -2.6214e-18,\n 1.5359e-19, -6.5940e-19, -1.0217e-16, -3.1662e-18, 2.0284e-18,\n -1.9014e-17, -5.3030e-18, -8.7467e-17, 4.4510e-19, -1.0555e-19,\n -9.1738e-17, -2.7322e-17, 6.7408e-20, 5.6630e-19, -2.2847e-19,\n -8.7213e-17, -5.8745e-19, -3.1113e-18, -3.7973e-17, 5.7974e-19,\n -7.2788e-18, -2.1383e-16, -6.4284e-19, -1.5794e-17, -9.9281e-17,\n -5.2228e-17, 2.3297e-19, -5.8756e-18, -1.2024e-17, -9.3042e-20,\n -1.1480e-16, 7.7332e-19, -3.3748e-19, -1.7577e-19, 4.0776e-19,\n 2.5487e-19, 1.5846e-18, -1.1464e-18, 4.8797e-19, 3.7621e-19,\n -1.7783e-16, 1.4909e-18, 1.6930e-18, 1.2098e-18, -2.4064e-19,\n -2.3547e-18, -1.1306e-16, 5.3459e-18, 6.3793e-19, -3.7955e-19,\n -4.2391e-20, -1.2709e-16, -5.9693e-18, 9.4267e-21, -8.6259e-20,\n -3.3225e-19, -2.3192e-16, 6.2429e-19, -7.3949e-19, -4.7128e-18,\n 2.4284e-19, -1.5297e-16, 1.5279e-19, 3.7513e-18, -1.2302e-16,\n -1.8611e-17, -1.2330e-17, -1.0498e-18, -1.7892e-16, -1.9002e-17,\n 1.0873e-19, -6.0227e-17, -1.0846e-17, -7.9817e-17, 7.3852e-19,\n -1.1598e-16, -8.1928e-17, -1.7402e-18, 7.3776e-20, -2.4683e-17,\n 2.5413e-18, 3.7884e-19, -5.7322e-17, -5.2731e-19, -1.0830e-17,\n -1.5708e-18, 6.6933e-19, -3.7837e-19, -1.1261e-17, 1.9546e-18,\n -4.2083e-18, -1.5093e-18, -7.1192e-18, 3.2807e-19, -5.6651e-19,\n 8.7553e-19, -1.3985e-19, 1.4309e-19, -7.9551e-17, 6.9676e-19,\n 4.4798e-19, -2.3894e-21, -9.5245e-17, -4.5924e-17, -6.5209e-17,\n 4.1672e-20, -1.4704e-16, -9.6314e-18, -9.8701e-17, -7.1233e-17,\n -3.6537e-19, -1.5476e-16, 6.9175e-20, -8.2402e-17, -5.6987e-17,\n -2.0829e-19, 8.6302e-19, -2.6959e-18, -1.9246e-19, -3.1000e-18,\n -7.9894e-18, -2.5514e-17, 4.6175e-18, -2.7141e-19, -1.1350e-18,\n -1.3052e-19, -5.4061e-19, -1.5749e-16, -1.3211e-16, -1.6333e-17,\n -2.0896e-17, -6.4606e-19, 2.3477e-20, 6.7138e-19, 6.7336e-19,\n 9.2482e-21, -7.7035e-17, 3.7188e-18, -4.4165e-17, -3.7884e-18,\n -1.6700e-17, -1.2558e-18, -1.4506e-17, -2.4645e-17, -5.6198e-19,\n 7.3041e-19, -2.3162e-20, 1.1188e-17, 2.6789e-20, 1.7502e-19,\n 2.0121e-19, -5.3157e-17, 5.5001e-19, -4.5115e-17, -2.7611e-18,\n -2.0232e-18, -1.9718e-19, 1.1532e-18, 6.2298e-20, -5.0947e-17,\n 2.6503e-20], device='cuda:0')", + "exp_avg_sq": "tensor([4.8706e-09, 6.0767e-10, 5.7377e-12, 1.9773e-08, 6.0160e-11, 5.0898e-11,\n 2.4024e-09, 1.2421e-11, 5.3312e-12, 3.4473e-08, 4.7995e-10, 2.6190e-09,\n 6.2928e-08, 2.1348e-09, 1.0447e-11, 8.4411e-09, 7.4260e-12, 1.1741e-11,\n 1.3027e-11, 5.4202e-11, 1.4193e-12, 4.6036e-13, 3.6780e-11, 4.5243e-09,\n 7.4948e-12, 1.7468e-07, 2.7943e-12, 6.0948e-08, 2.3877e-08, 4.4949e-11,\n 2.5297e-12, 1.6839e-08, 1.6654e-11, 4.9404e-10, 5.5692e-13, 1.3589e-11,\n 5.1099e-11, 6.5057e-09, 8.0070e-12, 7.4395e-11, 3.7876e-12, 1.1205e-11,\n 7.1153e-12, 1.7477e-12, 2.5094e-09, 2.4955e-10, 2.6719e-08, 6.7612e-08,\n 4.8835e-09, 2.1269e-09, 2.5224e-08, 1.8661e-11, 2.3184e-12, 1.5193e-10,\n 8.6027e-09, 1.7889e-09, 1.9511e-11, 6.8282e-13, 3.5423e-09, 4.0930e-08,\n 1.4805e-10, 1.4827e-12, 5.4156e-09, 3.8850e-08, 1.3148e-11, 1.8000e-11,\n 7.9024e-11, 5.9144e-10, 5.3950e-09, 1.7204e-10, 5.5830e-12, 2.9279e-11,\n 1.5593e-13, 8.5863e-12, 7.1270e-08, 2.3801e-12, 2.3477e-13, 6.9851e-13,\n 2.0278e-11, 6.8910e-10, 2.8683e-13, 1.8383e-12, 3.9972e-09, 3.1108e-13,\n 1.3143e-10, 1.3039e-11, 4.8954e-09, 9.9378e-10, 2.7968e-10, 9.6964e-09,\n 6.8779e-09, 7.1016e-10, 6.5982e-12, 1.8287e-12, 2.7539e-10, 2.0099e-10,\n 2.3651e-09, 4.2162e-09, 9.3854e-13, 1.6553e-12, 1.2517e-09, 7.9555e-12,\n 6.6010e-08, 1.3649e-11, 9.2259e-10, 6.6852e-08, 6.3377e-12, 8.5701e-13,\n 7.0039e-11, 4.2368e-11, 1.8761e-08, 1.3308e-11, 8.4567e-14, 6.2056e-09,\n 1.0766e-10, 2.8528e-09, 1.1311e-07, 6.9091e-11, 3.2284e-09, 1.1620e-08,\n 1.8970e-08, 1.1310e-11, 4.4835e-13, 3.4244e-12, 6.1918e-11, 3.7665e-08,\n 1.8234e-12, 2.3640e-09, 7.4062e-12, 7.3107e-09, 1.1059e-11, 5.4218e-11,\n 3.4491e-13, 1.0117e-09, 6.6110e-12, 3.1213e-08, 1.1909e-10, 3.7052e-12,\n 6.1524e-13, 5.4955e-12, 5.1453e-10, 2.6832e-08, 3.7695e-13, 1.8499e-11,\n 2.6795e-11, 8.4834e-11, 3.1363e-08, 3.3599e-10, 1.8825e-10, 8.0185e-11,\n 1.0446e-11, 1.8745e-07, 5.0195e-14, 2.3276e-10, 2.5209e-12, 2.0902e-10,\n 5.5297e-08, 2.2039e-12, 2.9672e-11, 1.1728e-08, 6.2617e-09, 9.1415e-12,\n 5.8896e-12, 9.4439e-08, 1.9811e-09, 2.2261e-11, 1.7299e-09, 6.8398e-09,\n 3.1386e-08, 2.9170e-10, 1.8437e-08, 1.1245e-08, 2.2157e-09, 5.6009e-11,\n 2.0094e-09, 5.5161e-14, 2.0503e-11, 5.9734e-09, 6.8849e-11, 1.3212e-09,\n 8.2251e-09, 7.0979e-12, 3.9760e-11, 4.3191e-09, 1.1859e-12, 4.3813e-09,\n 3.2033e-11, 8.1016e-10, 1.2332e-10, 2.9073e-10, 4.4122e-13, 1.0632e-13,\n 2.5235e-12, 1.4611e-08, 2.0256e-10, 3.1374e-11, 9.7382e-11, 1.5417e-08,\n 3.9856e-10, 1.9678e-08, 6.8803e-14, 1.5512e-07, 5.5727e-09, 6.4378e-09,\n 2.7044e-09, 1.8409e-09, 2.7471e-08, 5.3423e-13, 1.1035e-08, 1.7954e-09,\n 6.6054e-09, 8.8991e-12, 1.0705e-13, 8.4434e-13, 1.2552e-10, 1.5856e-09,\n 1.6268e-09, 1.7203e-12, 1.6021e-11, 2.0647e-13, 8.8272e-10, 1.3082e-09,\n 4.5123e-08, 2.6396e-08, 4.0114e-09, 7.0306e-09, 5.5481e-13, 9.2710e-11,\n 5.0995e-12, 2.7957e-10, 3.4222e-09, 1.5815e-08, 9.4300e-13, 1.5655e-10,\n 2.6380e-14, 5.8000e-09, 1.1017e-10, 3.4289e-08, 1.0647e-11, 1.0411e-10,\n 1.2825e-10, 2.5216e-09, 7.7512e-12, 6.6937e-11, 9.8843e-11, 2.4262e-11,\n 1.4604e-10, 1.3007e-11, 3.3324e-10, 2.2806e-10, 1.4690e-13, 2.1810e-10,\n 9.0287e-11, 1.9121e-11, 5.9688e-10, 2.0826e-10], device='cuda:0')" + }, + "39": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.5955e-17, 3.4725e-18, 2.6206e-19, -6.9099e-17, -3.6511e-18,\n -6.8887e-18, -4.4725e-17, -4.0320e-20, -4.5145e-17, -9.0825e-17,\n -6.1042e-17, 1.7278e-18, -8.4299e-17, -2.9262e-17, -3.3036e-18,\n -5.0692e-17, -4.8658e-17, 2.1531e-18, 7.3016e-18, -1.3096e-17,\n 1.0142e-18, -1.2778e-18, -7.4881e-18, -3.6693e-17, -2.5435e-20,\n -1.1009e-16, 2.0982e-18, -9.9212e-17, -7.9601e-17, -8.1358e-19,\n 2.3822e-18, -8.8427e-17, -2.3184e-17, 1.1544e-18, -1.3078e-18,\n 3.5968e-19, -7.9164e-18, -8.7351e-18, -1.1422e-18, -6.4035e-19,\n -5.6820e-18, -4.6382e-17, 3.3949e-18, 4.2838e-18, -4.1368e-17,\n 3.3820e-19, -4.7278e-17, -9.5365e-17, -1.2613e-17, -3.7776e-17,\n -4.5455e-17, -3.4724e-20, -3.1708e-17, -6.0159e-18, -4.6259e-17,\n -3.1627e-17, -9.6685e-19, 2.1022e-18, -5.6394e-17, -8.2378e-17,\n -1.1891e-18, 5.7278e-20, -8.9071e-17, -1.0625e-16, -2.6438e-17,\n -1.1115e-18, -6.6980e-18, 8.5854e-19, -4.2359e-17, 2.3387e-18,\n -3.7671e-17, 7.8473e-18, -1.4311e-18, -1.6944e-19, -7.1484e-17,\n -2.8243e-18, 2.5047e-18, -3.5739e-18, 8.6068e-19, 3.0211e-18,\n 4.2879e-18, -3.8807e-17, -2.1595e-17, 9.9578e-19, -5.8058e-17,\n -4.4631e-18, -3.3434e-17, 6.0233e-18, -8.8547e-19, -5.5848e-17,\n -2.9307e-17, -9.3920e-19, 3.7864e-18, -1.3616e-18, 1.5466e-18,\n -8.1021e-18, 3.5668e-18, -8.2093e-17, 2.1823e-19, -1.1773e-17,\n -2.6838e-17, -1.3980e-17, -4.6980e-17, -3.9274e-18, -6.7320e-19,\n -4.8523e-17, -5.6939e-17, 2.1523e-19, -1.0519e-17, 4.6223e-18,\n -5.5180e-17, 3.9371e-18, 2.3736e-18, -3.3563e-17, -1.2475e-17,\n -2.8956e-17, -9.6393e-17, 3.5337e-18, -3.0351e-17, -6.0869e-17,\n -5.6064e-17, -1.9059e-18, 4.3700e-18, -4.2298e-17, 3.0228e-18,\n -6.5851e-17, -5.1703e-19, -4.0916e-18, 1.6400e-19, -4.2727e-18,\n 3.0873e-18, -1.6682e-17, 7.9977e-19, -4.9146e-18, -2.3546e-18,\n -1.0493e-16, -9.1343e-18, -6.0763e-19, 7.0204e-20, 2.9154e-18,\n -1.7505e-17, -7.3304e-17, -3.8633e-18, -1.5440e-17, 1.6573e-19,\n -8.2837e-18, -7.3422e-17, 3.1390e-18, -2.4583e-18, -1.7006e-18,\n -1.3577e-18, -1.0937e-16, -4.9120e-19, 1.9616e-17, 4.0752e-18,\n 1.0459e-19, -9.8337e-17, -7.0724e-18, -2.9128e-18, -8.0976e-17,\n -4.2949e-17, 9.1268e-18, -7.9139e-19, -8.7263e-17, -2.2157e-17,\n -2.3261e-18, -5.5111e-17, -2.1172e-17, -4.5615e-17, -4.4551e-18,\n -7.8038e-17, -5.8475e-17, -8.8256e-18, 1.0845e-18, -3.7009e-17,\n -1.8097e-18, -2.2230e-18, -4.8114e-17, 8.9709e-18, -1.7312e-17,\n -1.1588e-17, -2.5367e-18, 3.3999e-18, -2.9513e-17, -1.5046e-18,\n -3.2043e-17, -7.7595e-20, -2.5501e-17, 3.5480e-18, -1.7183e-18,\n -6.7961e-19, -1.5531e-18, -1.6936e-19, -5.7362e-17, 7.6845e-19,\n -7.6829e-18, -5.4565e-18, -5.9562e-17, -7.4763e-17, -4.6272e-17,\n 1.8185e-18, -7.6088e-17, -2.2079e-17, -8.5849e-17, -6.4576e-17,\n 1.1206e-18, -8.9014e-17, -2.6880e-19, -5.9291e-17, -5.9455e-17,\n -1.3201e-18, -6.1133e-18, 1.8120e-18, -6.5295e-19, -1.7457e-17,\n -7.7801e-18, -2.8996e-17, -3.8150e-18, 1.6730e-18, 8.6713e-19,\n -7.4429e-19, -1.9348e-17, -7.6489e-17, -8.6433e-17, -4.1393e-17,\n -4.9471e-17, 1.1687e-19, -2.1838e-19, -8.6465e-18, -4.9187e-18,\n -4.2356e-18, -5.2334e-17, -2.6447e-18, -6.0661e-17, 2.7298e-18,\n -2.6351e-17, 5.6426e-18, -2.6601e-17, -4.8264e-17, 3.4086e-19,\n -5.6254e-18, -4.9155e-19, -8.5208e-18, 6.1134e-19, 5.3941e-18,\n -4.0487e-18, -7.1031e-17, -2.1284e-18, -5.8588e-17, 4.1222e-19,\n 1.0669e-18, -4.4939e-19, -7.2939e-18, -4.6566e-18, -6.2350e-17,\n -3.1599e-20], device='cuda:0')", + "exp_avg_sq": "tensor([4.9399e-09, 2.6446e-10, 3.0074e-10, 1.2316e-08, 2.0635e-11, 1.3214e-10,\n 3.7195e-09, 4.4743e-12, 4.1074e-10, 1.7854e-08, 4.1589e-09, 5.7726e-09,\n 2.6945e-08, 3.0919e-09, 1.6342e-10, 9.9864e-09, 1.0495e-09, 4.6617e-10,\n 3.0163e-10, 2.7315e-11, 9.8341e-12, 1.1798e-13, 1.5215e-11, 7.5797e-09,\n 1.9207e-12, 6.8212e-08, 1.3494e-10, 4.3503e-08, 1.1163e-08, 6.0521e-10,\n 3.7659e-10, 1.8244e-08, 2.2390e-09, 2.1584e-09, 2.6306e-10, 3.9017e-10,\n 2.2137e-11, 1.1525e-08, 4.7183e-12, 3.4869e-11, 9.8179e-13, 1.1800e-09,\n 1.4902e-12, 1.2094e-13, 3.8678e-09, 8.8228e-09, 9.1046e-09, 3.7218e-08,\n 3.4873e-09, 3.9761e-09, 1.1428e-08, 1.0640e-09, 5.5014e-10, 7.2319e-11,\n 5.2023e-09, 1.2297e-09, 6.3633e-12, 4.2500e-11, 8.2544e-09, 1.4807e-08,\n 5.9548e-10, 5.5179e-12, 2.0335e-08, 2.9642e-08, 5.4388e-11, 1.1474e-11,\n 3.7494e-11, 3.8098e-09, 4.6500e-09, 1.3249e-09, 5.6673e-10, 1.5306e-11,\n 1.6142e-10, 5.8531e-10, 2.9201e-08, 1.5884e-13, 3.0416e-11, 3.2962e-11,\n 6.2232e-10, 2.6811e-10, 4.6264e-11, 2.8586e-10, 4.3479e-09, 2.6986e-12,\n 2.0213e-09, 2.3670e-10, 3.3576e-09, 1.7364e-09, 9.4554e-11, 5.4948e-09,\n 2.1877e-08, 1.7759e-09, 2.4055e-11, 8.0865e-13, 1.4127e-10, 9.2266e-11,\n 6.8999e-09, 8.1725e-09, 2.3333e-10, 6.0471e-10, 4.1203e-09, 1.8877e-09,\n 3.2146e-08, 6.5851e-12, 3.9714e-09, 2.9807e-08, 8.1042e-10, 2.9081e-13,\n 3.5017e-11, 2.1623e-11, 1.2136e-08, 5.7996e-13, 1.3761e-12, 7.2835e-09,\n 5.7224e-11, 3.6914e-09, 3.2534e-08, 6.6425e-12, 2.5373e-09, 8.3697e-09,\n 8.4108e-09, 2.4431e-10, 3.0777e-11, 4.1283e-10, 1.7822e-10, 2.3262e-08,\n 3.7179e-10, 2.4994e-09, 4.5427e-10, 1.3578e-08, 5.3134e-12, 2.6876e-11,\n 1.2932e-10, 3.0213e-09, 1.8144e-12, 2.0318e-08, 1.1261e-09, 2.3281e-13,\n 2.3102e-10, 2.2712e-11, 5.2957e-10, 2.5757e-08, 1.9811e-12, 8.5097e-12,\n 2.3266e-12, 4.3463e-11, 2.3233e-08, 1.1508e-10, 7.4873e-11, 3.5759e-10,\n 1.4439e-12, 7.1295e-08, 2.0420e-12, 1.0703e-10, 6.6035e-12, 2.7945e-09,\n 5.1021e-08, 3.8281e-10, 1.0972e-10, 1.1143e-08, 3.5227e-09, 2.0815e-11,\n 3.3159e-12, 3.7710e-08, 6.5637e-09, 1.0761e-11, 3.6297e-09, 3.6203e-09,\n 1.2843e-08, 1.4566e-09, 1.8892e-08, 1.0873e-08, 1.8592e-09, 2.9376e-11,\n 2.7796e-09, 5.7400e-12, 1.0297e-11, 9.8746e-09, 3.3673e-11, 5.3065e-09,\n 7.6550e-09, 2.4269e-12, 7.2622e-10, 3.2706e-09, 2.2201e-10, 6.9527e-09,\n 1.0929e-11, 1.4611e-09, 5.3336e-11, 5.0011e-09, 3.1327e-11, 1.1019e-10,\n 1.3085e-10, 2.0328e-08, 2.4328e-09, 1.5257e-11, 4.0316e-11, 1.0550e-08,\n 4.1555e-09, 8.5845e-09, 1.8066e-11, 7.2777e-08, 3.1122e-09, 1.6000e-08,\n 9.1822e-09, 4.3805e-09, 1.9373e-08, 3.9587e-14, 1.2494e-08, 1.1768e-08,\n 4.9695e-09, 3.2822e-12, 1.0150e-11, 1.5038e-12, 1.1516e-09, 7.3914e-09,\n 3.8382e-09, 1.7157e-12, 3.1407e-12, 1.3016e-12, 3.6172e-09, 2.4729e-09,\n 1.5699e-08, 2.6359e-08, 3.8964e-09, 7.7542e-09, 8.3171e-11, 5.7961e-10,\n 6.6128e-10, 8.3526e-10, 6.2370e-09, 1.6016e-08, 9.0142e-10, 5.4206e-09,\n 1.7592e-11, 3.2517e-09, 4.5740e-11, 1.4318e-08, 2.2604e-09, 3.3031e-11,\n 6.5346e-10, 1.0791e-08, 6.3219e-10, 3.5167e-11, 4.6886e-11, 1.1209e-11,\n 5.1236e-09, 6.0387e-12, 2.4605e-09, 7.8115e-11, 1.6160e-11, 3.0930e-09,\n 9.5012e-10, 8.9806e-12, 8.5843e-09, 1.0874e-09], device='cuda:0')" + }, + "40": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.0820e-21, -5.5492e-20, -1.2515e-19, ..., -1.3888e-19,\n 5.2068e-20, 2.7703e-20],\n [ 1.9885e-19, -1.1317e-19, 4.6576e-19, ..., -2.1118e-19,\n -2.0113e-20, -2.5635e-20],\n [ 8.9313e-19, 6.8196e-19, -5.0838e-19, ..., 1.2183e-18,\n 5.8167e-19, 5.9699e-19],\n ...,\n [ 4.7636e-19, 2.6065e-19, 6.3321e-20, ..., 1.0807e-18,\n 2.2634e-19, 3.9116e-19],\n [ 9.3777e-19, 4.2981e-19, 8.8546e-19, ..., 2.6479e-18,\n 4.3206e-19, 2.6070e-19],\n [ 2.5997e-20, 7.9920e-21, 6.1412e-20, ..., 1.0207e-19,\n 1.0906e-20, 7.2758e-20]], device='cuda:0')", + "exp_avg_sq": "tensor([[9.2120e-12, 2.5619e-12, 8.5793e-12, ..., 1.0320e-11, 1.0842e-11,\n 1.4258e-11],\n [3.3350e-13, 5.2916e-13, 2.7526e-13, ..., 3.0981e-13, 1.7664e-12,\n 7.4584e-13],\n [5.0389e-13, 6.9471e-14, 2.3474e-13, ..., 7.7545e-14, 4.8315e-13,\n 2.1767e-13],\n ...,\n [3.6149e-13, 3.9595e-14, 2.3526e-13, ..., 5.2738e-14, 2.8306e-13,\n 1.0759e-13],\n [2.8449e-11, 7.7368e-12, 2.0398e-11, ..., 2.2804e-11, 3.4983e-11,\n 3.9117e-11],\n [6.1138e-12, 1.8130e-12, 6.0504e-12, ..., 2.2600e-12, 8.1708e-12,\n 6.3811e-12]], device='cuda:0')" + }, + "41": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-2.6087e-17, 1.3653e-18, 3.4616e-16, -1.6394e-17, 1.1837e-17,\n 1.0849e-17, 1.7252e-16, 3.9334e-16, -5.8777e-16, -6.3486e-16,\n 4.4750e-17, -1.3593e-16, -6.2363e-16, 5.7042e-17, 5.7414e-16,\n -6.6723e-16, -1.3276e-15, -1.7646e-17, 3.9009e-16, 5.1482e-17,\n -1.4131e-16, 1.9756e-17, 9.8101e-17, -2.8069e-16, -4.8063e-17,\n -1.9255e-15, -2.5371e-16, -1.6717e-15, -1.3343e-15, -5.8949e-16,\n 3.5701e-16, -1.2194e-15, 1.1054e-15, 5.2351e-17, -1.1181e-16,\n 2.7070e-16, -1.6581e-16, -1.2665e-17, 6.5922e-16, -2.6123e-16,\n 2.4160e-17, -4.0971e-16, 4.7047e-16, 7.2887e-16, -7.4094e-17,\n 6.2207e-16, -2.1211e-17, -1.2860e-15, 3.9991e-16, -4.9583e-16,\n -1.3814e-17, -8.3831e-17, -7.4653e-16, 1.0095e-15, 3.1747e-17,\n 1.6493e-16, 1.3254e-15, -6.3704e-16, -1.8878e-16, 6.0919e-16,\n 3.5639e-16, 6.2294e-16, -1.1990e-15, -4.3085e-17, -3.5258e-17,\n 2.5745e-16, -3.6396e-16, -3.5443e-17, 6.0151e-17, -2.6835e-16,\n 1.6364e-17, -1.3114e-16, 7.8128e-16, -2.0750e-16, 9.3317e-16,\n -2.1127e-16, 1.4300e-17, 4.4334e-16, 1.1464e-15, -1.0055e-15,\n 2.2654e-16, 5.3220e-17, 1.8659e-17, -4.6127e-16, 8.1538e-17,\n -1.8665e-16, 1.4625e-16, -3.6331e-16, 1.0198e-15, -1.8921e-16,\n 1.3398e-15, -2.0749e-16, -9.9418e-16, 2.0422e-16, 1.5643e-16,\n -2.6202e-16, -1.6843e-16, 2.6099e-16, -8.7096e-16, 1.4529e-16,\n 5.8989e-17, -1.3292e-16, -9.8005e-17, 6.5815e-16, 1.0869e-17,\n -4.8075e-16, 2.4936e-17, -2.5815e-16, -9.5715e-16, -2.0397e-16,\n 5.3701e-17, -5.1681e-16, 8.8888e-17, 3.9268e-17, -4.4189e-17,\n -3.7037e-16, -3.5506e-16, 6.6835e-16, -6.9533e-18, -5.2662e-17,\n -1.9773e-17, 3.1225e-16, 6.3200e-16, -1.6558e-16, 1.5689e-17,\n -3.3744e-16, 1.1077e-15, -5.0491e-17, 3.6660e-16, 5.7452e-16,\n 8.1035e-17, -7.3936e-16, 2.5943e-16, -1.6802e-16, -3.2667e-17,\n -1.7728e-15, 6.7361e-16, -2.6764e-17, -2.2348e-16, -4.1371e-16,\n 4.8953e-16, 1.2039e-15, 6.8691e-16, -2.6520e-17, -1.8315e-16,\n -1.7183e-17, -4.6469e-16, -6.7023e-16, 1.3578e-16, 2.4401e-16,\n 1.6086e-16, -1.8810e-15, 4.1062e-16, 2.2633e-16, 2.1874e-16,\n 5.0316e-16, 2.2921e-17, 4.7582e-16, 3.4999e-16, -5.7500e-17,\n 2.3837e-16, -1.5596e-15, 9.7003e-18, 8.6820e-19, -8.2056e-17,\n 1.9232e-16, -5.2392e-17, 1.7145e-16, 8.5440e-17, 2.2511e-17,\n -5.3152e-17, -6.1380e-16, 2.2234e-16, 3.8380e-16, -2.6607e-17,\n 2.4022e-18, -4.3034e-17, -7.2471e-17, 2.2205e-16, 4.3221e-16,\n 1.4226e-16, 7.4294e-17, -3.0462e-16, -4.0079e-17, -1.5389e-16,\n 8.6221e-17, -2.3587e-17, 8.7204e-16, -1.4743e-16, 3.2333e-16,\n 7.5499e-16, 8.6210e-16, 5.8003e-16, -2.8306e-17, 7.8418e-17,\n -1.4354e-16, 4.3820e-16, 3.5506e-16, -1.3038e-15, 1.2747e-15,\n -3.0127e-16, -2.0430e-16, -1.0931e-16, -1.2592e-15, -1.4792e-16,\n -3.4915e-17, -4.4308e-16, -1.3231e-20, 2.3374e-16, 3.4142e-17,\n 7.3225e-17, 1.9904e-16, 6.2600e-16, -2.2633e-16, 4.9283e-17,\n 3.7920e-16, 2.4572e-16, 5.7785e-16, 6.4460e-16, 3.9220e-16,\n 5.7603e-17, 7.5995e-17, -8.0538e-17, -7.4008e-17, 4.7089e-17,\n -1.5983e-17, 2.6204e-18, 7.3668e-17, -1.2812e-15, 3.0214e-17,\n 2.3916e-16, -1.9835e-16, -3.6504e-16, 4.5577e-17, 1.4350e-16,\n 6.7491e-17, 3.9381e-16, -2.3767e-17, 4.6859e-16, 5.6357e-16,\n -2.5245e-16, 5.0873e-16, -5.8774e-16, 4.9710e-17, -2.0216e-16,\n 2.0134e-16, -6.7181e-17, 2.2099e-16, 2.3548e-16, -9.9071e-17,\n 1.9428e-16, -6.5936e-18, 2.2597e-16, 2.0463e-16, 1.4440e-16,\n 5.3873e-17], device='cuda:0')", + "exp_avg_sq": "tensor([2.1366e-06, 1.7041e-07, 8.2555e-08, 4.7756e-08, 5.8037e-06, 1.3290e-06,\n 1.3390e-07, 5.0650e-06, 1.5150e-05, 2.8646e-06, 1.9752e-09, 1.1201e-05,\n 1.5493e-05, 4.0934e-07, 5.0346e-07, 2.0818e-05, 4.6771e-05, 1.4301e-08,\n 1.5637e-06, 3.7884e-08, 1.8527e-05, 1.1694e-07, 1.9219e-06, 3.8411e-07,\n 5.5809e-07, 8.4144e-05, 2.5484e-06, 3.9919e-06, 4.1937e-06, 1.8334e-06,\n 9.4094e-05, 1.9398e-05, 1.3748e-05, 7.2254e-07, 4.5282e-06, 2.1594e-05,\n 3.1931e-07, 4.5210e-05, 7.5641e-06, 1.5209e-06, 1.1618e-06, 3.7796e-07,\n 1.9573e-06, 2.4213e-07, 1.0971e-05, 1.0395e-04, 9.3219e-09, 2.0838e-05,\n 9.6139e-06, 1.2655e-05, 6.5004e-07, 2.9643e-08, 2.4213e-05, 5.8791e-05,\n 2.9327e-08, 3.3090e-06, 1.6102e-06, 3.3433e-06, 2.3050e-07, 1.3333e-05,\n 4.1532e-07, 7.9854e-07, 3.1549e-05, 3.2175e-07, 1.3718e-07, 3.2655e-05,\n 1.3641e-06, 4.8349e-06, 1.3272e-07, 4.4831e-06, 3.2116e-07, 2.3915e-06,\n 1.0360e-05, 2.4859e-07, 3.9731e-05, 6.2885e-07, 6.3447e-08, 1.1791e-06,\n 3.6398e-05, 2.4273e-05, 9.4607e-08, 4.2094e-08, 2.2935e-06, 4.3812e-07,\n 3.6393e-06, 6.4898e-07, 1.7274e-07, 1.1866e-07, 7.1383e-07, 8.0592e-07,\n 5.7753e-05, 1.6271e-08, 6.0784e-06, 4.3147e-08, 3.9702e-05, 9.4387e-07,\n 3.2271e-06, 1.3897e-06, 7.3028e-06, 3.1308e-06, 4.7244e-07, 2.6180e-07,\n 2.7321e-08, 8.3393e-07, 6.1655e-06, 7.2769e-06, 3.5603e-06, 2.5684e-07,\n 1.0567e-05, 6.3096e-07, 1.9701e-06, 3.3652e-07, 1.6276e-06, 1.1894e-08,\n 3.4509e-07, 2.3690e-06, 4.1471e-06, 5.6074e-07, 1.4100e-06, 1.9515e-07,\n 2.7338e-07, 7.4846e-06, 1.7735e-06, 1.0592e-05, 7.0327e-07, 1.1291e-08,\n 2.8662e-05, 2.6913e-08, 1.3700e-05, 6.1107e-05, 1.1026e-06, 1.0519e-05,\n 5.4504e-07, 1.1959e-06, 1.0199e-05, 3.0515e-06, 3.9690e-07, 8.0671e-07,\n 3.4571e-07, 3.0392e-07, 5.4178e-06, 1.5359e-05, 2.0132e-07, 2.1782e-08,\n 1.9268e-07, 3.4039e-08, 1.1260e-05, 1.5941e-05, 1.4179e-07, 2.2791e-07,\n 6.7217e-06, 3.2593e-05, 2.0064e-06, 5.7802e-08, 1.4183e-05, 1.3661e-05,\n 6.7792e-05, 3.0373e-06, 5.8431e-05, 4.0834e-08, 5.6203e-06, 2.7057e-05,\n 2.6187e-07, 3.1562e-08, 8.8073e-06, 7.4663e-06, 4.4954e-07, 4.5364e-06,\n 5.6391e-07, 1.5971e-08, 3.5410e-07, 3.1057e-06, 1.1578e-05, 1.8302e-05,\n 2.0592e-06, 7.6319e-07, 1.4755e-08, 7.4113e-07, 3.3609e-08, 1.5261e-05,\n 1.9863e-07, 1.5571e-07, 2.2597e-06, 6.2916e-08, 2.4354e-08, 7.2959e-06,\n 2.2961e-07, 3.3015e-06, 2.7878e-06, 4.0569e-05, 2.3198e-05, 9.4098e-06,\n 6.1222e-07, 2.1128e-07, 8.9287e-09, 3.2011e-06, 1.0289e-05, 4.8150e-06,\n 1.0932e-05, 4.1666e-06, 5.0438e-07, 1.3934e-06, 1.6700e-08, 3.2230e-05,\n 1.2001e-06, 2.3568e-08, 1.2437e-05, 1.7338e-08, 3.2563e-07, 8.2149e-07,\n 1.6902e-08, 2.0953e-07, 6.5598e-06, 1.0948e-06, 1.3390e-08, 5.1294e-08,\n 1.3861e-06, 1.5465e-05, 4.7140e-07, 1.3669e-07, 1.3977e-05, 1.8385e-05,\n 1.1027e-08, 3.6149e-05, 3.5320e-08, 4.6660e-06, 6.4883e-07, 1.3721e-07,\n 9.4721e-06, 1.7904e-08, 1.7704e-05, 1.1712e-07, 6.8930e-07, 3.5159e-08,\n 9.4677e-07, 1.3919e-06, 3.8335e-05, 4.8853e-05, 9.4899e-06, 2.3043e-06,\n 4.8076e-07, 6.1138e-05, 8.1496e-07, 4.6819e-09, 1.6825e-06, 2.4399e-08,\n 3.2287e-07, 5.7202e-07, 4.8161e-06, 5.9557e-08, 2.3851e-07, 9.8095e-06,\n 6.2180e-07, 2.6107e-08, 6.1603e-06, 1.0500e-06], device='cuda:0')" + }, + "42": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-6.3008e-19, -2.9675e-18, 2.2799e-19, -1.2472e-18, -8.4098e-17,\n -1.1564e-19, 8.2962e-19, -3.2394e-18, -1.0283e-16, -1.1111e-16,\n -1.0714e-18, -2.0784e-19, -5.2976e-17, -1.5904e-19, -6.5596e-18,\n -1.2338e-16, -7.2507e-17, 4.1126e-19, -1.9796e-19, 9.1744e-19,\n -1.0515e-16, 6.4496e-19, 1.0120e-18, -2.0430e-17, -6.2962e-19,\n -1.3859e-16, 2.8791e-18, -3.2691e-17, -5.2985e-17, -4.6093e-17,\n -6.5964e-18, -8.2978e-17, -1.3017e-17, 1.1930e-20, 6.2564e-19,\n -2.5104e-20, -1.5165e-17, -2.8296e-18, -4.2670e-17, -9.5051e-19,\n 1.4410e-19, -3.2042e-17, -4.3834e-19, -5.3707e-19, -2.2903e-18,\n -9.7781e-19, -1.5516e-18, -1.6301e-16, -5.8389e-19, -1.0507e-16,\n -7.8111e-19, -7.0398e-19, -1.1618e-16, -5.5649e-19, 3.0368e-19,\n 1.6827e-19, 1.9693e-18, -5.4480e-17, -4.2848e-18, -9.2910e-17,\n -1.1873e-19, -5.3605e-19, -1.1934e-16, -2.7172e-18, -1.6538e-18,\n -2.4465e-18, -4.5434e-17, 3.1355e-20, -1.0790e-17, 1.4126e-18,\n -5.3537e-19, 1.2954e-18, -6.4904e-18, -2.9653e-18, -7.9584e-17,\n -8.4364e-19, -3.9040e-19, -2.6342e-19, 2.5367e-19, -7.8074e-17,\n 4.3643e-19, -2.2163e-19, 7.9086e-19, 2.3413e-18, -1.1367e-16,\n -1.9391e-18, 4.9971e-19, 2.5320e-18, -1.2422e-18, -6.0384e-17,\n -3.8667e-17, -3.8086e-19, -5.5600e-17, 4.6843e-19, -1.2119e-16,\n -1.9320e-17, -2.1246e-19, -9.0777e-17, -3.4973e-17, -7.9778e-17,\n -2.2885e-19, -1.7753e-18, -1.0087e-18, -2.0188e-17, -3.4310e-19,\n -3.8127e-17, -1.1620e-16, 6.1327e-19, -2.7925e-17, -1.6765e-18,\n -2.2060e-17, 5.9446e-18, -7.5169e-20, 1.0063e-19, -1.3748e-18,\n -2.2369e-17, -1.3716e-16, -4.9143e-19, -4.6447e-19, 6.0360e-19,\n 2.3376e-19, -8.5919e-20, -3.7108e-17, -4.2724e-17, -3.3779e-19,\n 3.3829e-19, 9.3151e-19, 7.8969e-21, 4.8161e-20, -4.4736e-19,\n 9.1340e-19, -1.3297e-16, 1.7858e-19, -2.3461e-18, -1.1057e-17,\n -7.9618e-17, -9.1182e-18, 5.4012e-20, 1.9722e-18, 3.1737e-18,\n -2.2940e-19, -5.7505e-17, -5.0568e-19, -8.8462e-19, -1.4727e-19,\n -5.0688e-18, -4.3916e-17, -1.2682e-16, 5.1665e-19, 3.8923e-19,\n 4.5510e-19, -8.1304e-17, 3.7456e-19, -6.5689e-19, -8.5053e-17,\n 1.1407e-19, -1.1633e-16, -7.4974e-18, -5.6772e-19, 1.2308e-18,\n 3.6455e-19, -1.3585e-16, -1.4184e-18, -3.9071e-19, -2.8520e-17,\n 5.3380e-19, -4.3517e-19, -4.5643e-19, -3.9985e-18, 3.7492e-19,\n -3.5181e-18, -4.3892e-17, -2.9647e-20, 3.1433e-19, -7.5411e-17,\n -1.1545e-20, 9.7559e-19, -1.6773e-18, -6.5434e-20, -2.7194e-17,\n 9.8122e-19, 8.8862e-19, 2.8470e-18, 1.2798e-18, 1.0719e-18,\n 4.3100e-19, 9.3991e-19, -8.7866e-18, 4.3292e-20, 4.2676e-19,\n -1.8806e-18, -2.0320e-17, -4.4652e-21, -6.7109e-18, 3.9910e-20,\n -3.0209e-17, -3.3680e-17, -4.5591e-17, -1.1834e-16, -4.7184e-17,\n 2.7038e-18, -8.7820e-18, -1.8699e-18, -1.8253e-16, -6.1433e-17,\n 8.2753e-19, -1.4889e-16, -1.1720e-19, 3.5691e-19, -9.5567e-18,\n 9.6325e-21, -1.0209e-17, -1.1071e-18, -1.1485e-18, -3.9302e-19,\n -4.4223e-19, 6.2325e-19, -1.1643e-18, -6.3956e-19, 5.9942e-20,\n 7.6025e-19, 6.7155e-19, -3.3472e-18, -6.2714e-17, 6.3981e-19,\n -1.2640e-18, -4.4786e-19, -5.9026e-20, -1.2675e-16, 8.6209e-19,\n 1.7281e-19, -1.6792e-18, 3.7002e-19, 9.0619e-19, -4.3462e-20,\n -1.1853e-19, -1.7645e-19, -9.5875e-18, -6.7877e-17, 1.5227e-19,\n -1.4890e-18, -1.2574e-19, 5.8757e-18, -2.8682e-20, 1.1641e-18,\n -2.7034e-19, -3.3746e-18, -1.3881e-19, -9.4386e-17, -7.0461e-18,\n 5.9361e-19, 2.8759e-20, -3.7737e-17, 7.7087e-19, -2.0834e-17,\n -1.5639e-19], device='cuda:0')", + "exp_avg_sq": "tensor([1.6174e-11, 5.3572e-10, 7.1966e-13, 2.5309e-10, 2.5288e-08, 8.1881e-12,\n 1.1098e-11, 1.1911e-09, 2.9092e-08, 2.4292e-08, 5.3630e-12, 1.8412e-11,\n 3.4238e-09, 1.4743e-10, 4.1124e-09, 4.8253e-08, 7.9066e-09, 6.6146e-13,\n 4.1646e-11, 5.6175e-11, 4.6120e-08, 1.0625e-11, 5.3983e-10, 2.2094e-11,\n 5.4743e-10, 5.8413e-08, 1.7233e-12, 1.8599e-11, 5.0876e-10, 9.0476e-09,\n 4.2112e-08, 1.3477e-08, 3.2111e-09, 4.7809e-11, 1.1213e-11, 1.7046e-09,\n 5.2470e-09, 1.0692e-08, 1.2940e-08, 2.7739e-12, 2.9700e-11, 1.4407e-10,\n 2.3906e-10, 8.1511e-12, 3.2657e-09, 7.4005e-09, 1.5963e-12, 7.6534e-08,\n 3.3969e-10, 2.5983e-08, 1.3174e-10, 5.4208e-13, 4.0933e-08, 2.0542e-08,\n 7.9220e-11, 1.1329e-12, 4.7500e-10, 1.5275e-09, 5.7308e-10, 6.3616e-08,\n 2.9167e-11, 6.9800e-11, 2.9674e-08, 6.1880e-10, 3.4040e-10, 7.9539e-09,\n 1.2325e-09, 3.2976e-11, 5.6892e-12, 4.1585e-12, 2.4463e-10, 4.6686e-13,\n 5.4564e-10, 8.9051e-13, 1.0450e-07, 9.8215e-14, 1.1943e-12, 2.8591e-11,\n 3.7115e-09, 8.1096e-09, 1.4163e-11, 1.3707e-10, 9.4163e-11, 3.9863e-12,\n 3.1672e-08, 4.0904e-13, 6.9245e-12, 2.1500e-13, 7.1848e-11, 4.1520e-09,\n 7.3591e-08, 8.6433e-13, 1.6085e-08, 7.3359e-13, 1.1743e-07, 1.9169e-09,\n 1.7859e-12, 2.3749e-08, 1.2193e-08, 1.4242e-08, 1.2828e-10, 3.0837e-10,\n 1.9518e-10, 2.0016e-09, 3.3890e-11, 8.5399e-10, 3.4837e-08, 1.0943e-14,\n 7.7727e-09, 2.8312e-13, 2.3672e-10, 1.2736e-12, 5.1943e-11, 3.5663e-12,\n 2.3903e-10, 3.5569e-09, 3.9787e-08, 1.8323e-10, 1.0162e-13, 9.1430e-11,\n 4.3518e-11, 3.2596e-10, 1.2529e-08, 3.2167e-09, 9.5859e-12, 9.5103e-12,\n 1.9273e-09, 3.5319e-12, 3.9492e-10, 1.2757e-08, 3.5554e-11, 4.6731e-08,\n 7.2391e-11, 3.9964e-13, 4.3855e-09, 2.8390e-09, 3.5462e-09, 1.2294e-10,\n 3.0926e-13, 1.1228e-13, 9.1171e-11, 4.5756e-08, 7.8713e-12, 2.3584e-12,\n 3.2337e-12, 2.1619e-12, 1.2124e-09, 4.2116e-08, 3.4108e-11, 4.3095e-13,\n 3.2287e-10, 6.1963e-09, 2.3851e-12, 3.0405e-11, 3.6171e-08, 8.6032e-10,\n 1.3929e-07, 3.7081e-10, 1.8830e-09, 1.2499e-12, 4.5286e-10, 4.9780e-08,\n 8.4861e-12, 2.4773e-10, 1.1898e-09, 5.0140e-11, 1.2526e-10, 1.0232e-09,\n 2.8666e-12, 2.3588e-12, 6.8839e-10, 6.0450e-10, 2.7728e-10, 1.3277e-09,\n 1.2396e-08, 1.0136e-13, 1.7003e-11, 2.9129e-10, 9.7982e-12, 4.6314e-09,\n 6.5961e-11, 1.3366e-11, 7.1047e-12, 1.0912e-11, 2.1565e-14, 1.9580e-10,\n 1.4493e-13, 4.4396e-09, 1.7591e-12, 5.2003e-09, 1.1409e-09, 7.1478e-09,\n 4.6560e-11, 9.2003e-10, 8.8847e-13, 3.2317e-09, 4.4376e-09, 5.9706e-09,\n 2.4363e-08, 3.1490e-08, 6.2629e-13, 2.0514e-11, 3.0776e-13, 9.9889e-08,\n 8.2758e-09, 3.8397e-12, 6.3832e-08, 2.0952e-13, 2.1265e-12, 3.6036e-12,\n 3.1207e-12, 4.4518e-09, 8.1074e-11, 6.3256e-13, 2.8994e-13, 1.1108e-12,\n 2.4800e-12, 3.6222e-10, 7.6052e-11, 2.7133e-11, 5.0518e-10, 8.7795e-10,\n 3.4610e-13, 2.1164e-08, 8.0412e-12, 3.2134e-11, 7.8723e-14, 1.6359e-11,\n 3.8418e-08, 1.6573e-12, 1.1767e-09, 3.5590e-12, 5.7517e-12, 2.0222e-12,\n 7.1808e-12, 3.3632e-13, 4.5226e-09, 2.0886e-08, 2.4197e-08, 5.2528e-11,\n 7.9793e-13, 9.5428e-09, 6.5836e-12, 2.0874e-13, 1.3168e-12, 2.5693e-12,\n 4.3184e-10, 9.7562e-12, 3.0276e-08, 6.6195e-12, 5.4485e-12, 6.2481e-11,\n 1.5273e-09, 2.8469e-12, 3.7977e-10, 9.9607e-11], device='cuda:0')" + }, + "43": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.2935e-18, 7.0668e-19, -5.0414e-19, -1.1593e-18, -5.1336e-17,\n -6.9544e-19, -4.0698e-18, -1.2054e-17, -6.3712e-17, -6.6878e-17,\n -1.3956e-17, 1.4431e-19, -5.1747e-17, -1.3740e-18, -1.2716e-17,\n -6.8972e-17, -6.8494e-17, -6.2935e-20, 4.3909e-18, -6.9455e-18,\n -5.7702e-17, -9.6113e-18, -8.1042e-18, -3.4728e-17, -1.5148e-17,\n -9.1463e-17, -1.8903e-18, -6.4119e-17, -6.3818e-17, -4.9379e-17,\n -1.4860e-17, -6.9597e-17, -1.2450e-17, -9.9699e-19, -3.5384e-19,\n -3.1890e-18, -2.9194e-17, -1.6979e-17, -3.1257e-17, 7.8395e-19,\n -2.1380e-18, -4.1619e-17, 5.0088e-18, 8.0263e-18, -1.8888e-17,\n 6.8043e-18, 4.2673e-19, -8.5431e-17, -4.9560e-18, -6.2538e-17,\n -3.8914e-19, 5.5508e-19, -6.8010e-17, -1.8581e-18, -3.8016e-18,\n -1.4080e-18, 4.3263e-18, -5.1952e-17, 3.2284e-18, -4.6233e-17,\n 2.9361e-19, 5.7755e-18, -7.6348e-17, 1.4424e-18, 1.0993e-19,\n -1.1696e-17, -4.5376e-17, -1.9003e-20, -2.3989e-17, -1.0679e-18,\n -1.3687e-18, -6.1280e-19, -1.0945e-17, 2.2377e-18, -3.9308e-17,\n 5.9506e-19, -3.8495e-19, 3.8122e-18, 5.9763e-19, -6.4036e-17,\n -2.8062e-18, -2.7313e-18, -4.3347e-18, -1.8444e-18, -5.6514e-17,\n 1.4630e-18, -6.5989e-18, -2.0440e-18, 1.4802e-17, -4.7576e-17,\n -2.2594e-17, 3.3061e-19, -5.8394e-17, -3.2773e-18, -5.7255e-17,\n -3.3251e-17, 1.0460e-19, -4.9299e-17, -4.9806e-17, -4.8261e-17,\n -2.0458e-19, 1.3134e-18, -8.8567e-18, -2.1126e-17, -4.6715e-19,\n -4.4681e-17, -5.7991e-17, -5.0523e-19, -4.8137e-17, 1.2697e-18,\n -3.0626e-17, -4.7211e-18, -1.6701e-18, -1.6952e-18, -2.2504e-18,\n -3.6510e-17, -6.7642e-17, 6.9467e-18, -2.4521e-19, -8.2468e-18,\n -9.9356e-18, -5.6512e-18, -2.8462e-17, -4.1576e-17, -4.4877e-19,\n -2.0506e-17, 7.5158e-18, -1.2122e-20, 1.6869e-19, -2.4516e-18,\n 8.4400e-19, -7.2402e-17, -3.1453e-18, 1.5980e-18, -2.3921e-17,\n -7.7350e-17, -1.4308e-17, -1.5912e-19, -1.8886e-18, -2.5766e-18,\n 1.6786e-18, -3.0233e-17, 6.3782e-18, -5.3448e-19, 1.1598e-19,\n -2.0096e-17, -4.6251e-17, -6.9570e-17, -6.8019e-18, -2.1978e-18,\n -3.3858e-18, -7.9283e-17, 1.3055e-18, 2.6463e-18, -4.8854e-17,\n 1.9698e-18, -5.8143e-17, -1.5086e-17, 3.2555e-18, -1.0824e-17,\n -2.7211e-18, -8.5111e-17, -1.6028e-17, -1.7659e-18, -3.4766e-17,\n -2.1137e-18, -4.0571e-18, -1.0002e-17, -1.6510e-17, -1.4816e-18,\n 2.7027e-18, -4.8721e-17, -4.0178e-18, -2.2297e-18, -4.9125e-17,\n -8.6569e-20, -1.1330e-17, 1.6619e-19, 4.6779e-19, -2.7713e-17,\n -5.1137e-18, -8.3362e-18, -2.2819e-18, -9.7750e-18, -7.3493e-19,\n -2.3138e-18, -7.3628e-18, -1.2335e-17, -3.5600e-20, -2.3340e-18,\n -5.0277e-18, -1.9633e-17, 4.4991e-18, 4.6608e-18, 9.0145e-19,\n -3.6960e-17, -2.9857e-17, -3.5536e-17, -7.7616e-17, -2.6102e-17,\n -2.1200e-18, -2.5958e-17, 1.5471e-18, -8.8331e-17, -4.7453e-17,\n -2.8141e-19, -7.1318e-17, -5.5213e-20, -6.2615e-18, -2.3324e-17,\n -2.5164e-19, -2.0870e-17, -4.9807e-18, 8.8704e-19, -6.6665e-19,\n -4.4358e-18, -3.5994e-18, 5.1482e-18, 5.5617e-18, 2.7806e-19,\n -3.8118e-18, -4.4739e-18, 2.6934e-18, -4.6155e-17, -2.6334e-18,\n 9.7975e-22, -1.3396e-19, -7.6874e-18, -7.9593e-17, -4.3117e-18,\n -1.3682e-18, -2.0189e-17, -2.9596e-19, -9.1673e-18, -1.5184e-18,\n -1.6839e-19, -1.9839e-18, -2.3923e-17, -4.1035e-17, -8.8575e-19,\n 1.1687e-18, -9.6781e-19, -4.5457e-18, -1.9204e-19, -9.4356e-19,\n 1.5495e-18, 1.9474e-18, -1.5370e-18, -5.0656e-17, -2.3253e-17,\n -4.5964e-18, 2.1894e-19, -3.4305e-17, -3.9762e-18, -2.7830e-17,\n -6.9613e-18], device='cuda:0')", + "exp_avg_sq": "tensor([8.1269e-10, 2.4511e-10, 4.3086e-13, 1.3942e-10, 1.0363e-08, 3.6273e-12,\n 6.3686e-12, 9.1110e-10, 2.1158e-08, 1.2098e-08, 2.4485e-10, 2.9599e-09,\n 1.4248e-08, 6.9034e-11, 3.0540e-09, 2.5925e-08, 3.0549e-08, 1.2900e-13,\n 1.9887e-11, 8.8615e-10, 2.4929e-08, 4.8076e-12, 1.3601e-09, 1.3785e-09,\n 1.7631e-09, 5.7808e-08, 2.5980e-10, 4.1485e-09, 5.6410e-09, 6.3279e-09,\n 2.3943e-08, 2.0301e-08, 1.1042e-08, 2.5179e-11, 5.8181e-10, 3.8211e-09,\n 3.9724e-09, 1.3083e-08, 9.9996e-09, 4.6699e-10, 2.8147e-10, 2.1008e-09,\n 1.0948e-10, 3.4519e-12, 4.5925e-09, 2.9046e-08, 5.4583e-11, 3.0238e-08,\n 4.1755e-10, 1.8051e-08, 6.0623e-11, 2.1219e-11, 2.6123e-08, 9.4827e-09,\n 4.6450e-11, 8.9425e-12, 2.0513e-09, 5.9063e-09, 2.9439e-10, 2.2289e-08,\n 1.5467e-11, 3.6793e-11, 2.8956e-08, 2.6846e-10, 1.7380e-10, 7.5756e-09,\n 4.2110e-09, 9.6031e-10, 1.0298e-09, 1.0451e-09, 1.2160e-10, 1.5052e-10,\n 2.8229e-10, 6.4216e-11, 3.7536e-08, 6.1597e-11, 4.6865e-11, 1.4878e-11,\n 2.3562e-09, 1.9844e-08, 7.5303e-12, 6.7870e-11, 8.7646e-10, 1.1404e-10,\n 1.3213e-08, 1.1051e-10, 3.4847e-12, 9.1045e-12, 3.9547e-11, 5.2647e-09,\n 3.6106e-08, 1.4762e-11, 9.9280e-09, 3.0444e-13, 3.9533e-08, 3.9278e-09,\n 9.2538e-10, 1.0219e-08, 6.3743e-09, 9.8692e-09, 6.6045e-11, 1.5481e-10,\n 9.4882e-11, 3.5764e-09, 1.0344e-09, 7.3771e-09, 1.3879e-08, 3.3435e-13,\n 6.7655e-09, 4.8310e-11, 3.1343e-09, 4.7880e-11, 4.6353e-10, 8.9016e-13,\n 1.0946e-10, 5.0857e-09, 1.5168e-08, 9.4489e-11, 1.5316e-11, 4.7897e-11,\n 2.3947e-11, 4.6880e-10, 5.2568e-09, 1.0637e-08, 2.7138e-10, 1.1730e-11,\n 2.3974e-09, 2.0013e-12, 1.6260e-09, 1.3319e-08, 1.6065e-11, 2.0821e-08,\n 3.7763e-11, 3.2390e-10, 2.5637e-09, 6.8920e-09, 4.0389e-09, 5.8341e-11,\n 1.5325e-11, 4.8486e-12, 1.8930e-10, 1.8730e-08, 3.5298e-12, 3.4319e-10,\n 1.0079e-10, 6.4377e-10, 1.0037e-08, 2.2429e-08, 1.8502e-11, 2.0220e-10,\n 1.5336e-09, 2.3692e-08, 2.2454e-10, 6.0872e-12, 2.0284e-08, 1.8214e-09,\n 5.5076e-08, 2.3226e-09, 1.4754e-08, 7.0202e-11, 1.5103e-09, 3.0632e-08,\n 3.6542e-10, 1.3304e-10, 7.6929e-09, 4.5828e-10, 5.8929e-11, 1.4733e-09,\n 8.0321e-10, 7.7358e-13, 3.0588e-10, 4.8441e-09, 7.3263e-10, 2.3905e-09,\n 7.6696e-09, 2.3544e-11, 6.7942e-12, 1.1831e-10, 2.1891e-12, 1.1044e-08,\n 3.4226e-11, 7.3460e-12, 2.9025e-10, 4.5630e-12, 1.1075e-14, 2.1333e-09,\n 2.5017e-10, 2.0580e-09, 3.8584e-10, 1.0557e-08, 9.0125e-10, 2.7406e-09,\n 2.3835e-11, 3.7848e-10, 5.3677e-12, 3.1641e-09, 1.0856e-08, 8.3240e-09,\n 1.7902e-08, 1.1883e-08, 2.3286e-11, 1.1403e-09, 4.4440e-12, 3.7391e-08,\n 5.6157e-09, 1.5529e-12, 2.3712e-08, 7.3110e-14, 3.1569e-10, 1.2619e-09,\n 4.2098e-13, 3.5476e-09, 8.8491e-11, 2.0545e-10, 2.5579e-12, 9.8766e-11,\n 2.8120e-10, 1.5919e-09, 4.1935e-11, 1.1526e-11, 3.0867e-09, 4.4772e-09,\n 4.4653e-12, 2.7108e-08, 2.6960e-12, 1.5924e-09, 3.8931e-11, 7.4200e-12,\n 1.8624e-08, 1.5702e-12, 4.0530e-09, 1.6976e-10, 2.1239e-10, 9.2194e-11,\n 3.1333e-12, 1.8860e-11, 8.8548e-09, 1.1253e-08, 1.4070e-08, 4.7559e-10,\n 9.5293e-11, 1.4544e-08, 1.9403e-10, 4.5596e-12, 2.4655e-10, 8.3091e-13,\n 1.9631e-10, 5.2361e-12, 1.4247e-08, 2.6444e-10, 2.8432e-12, 1.9242e-09,\n 3.9691e-09, 7.0960e-13, 5.5726e-09, 5.0644e-10], device='cuda:0')" + }, + "44": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-3.6876e-18, -1.1673e-18, 2.5406e-19, ..., -2.5257e-18,\n -8.4254e-19, -5.2654e-19],\n [ 4.9788e-19, -1.5828e-21, 5.1412e-19, ..., -8.3887e-19,\n 1.4239e-19, -1.1910e-19],\n [-2.5614e-19, 1.0274e-19, -1.8448e-19, ..., 5.3831e-19,\n -8.5500e-20, 2.7523e-20],\n ...,\n [-9.7502e-20, -7.2062e-20, -2.7935e-20, ..., 5.8753e-20,\n -8.9045e-20, -1.7336e-20],\n [ 5.2953e-19, 2.8279e-19, 1.7962e-19, ..., 1.1695e-18,\n 7.4714e-20, 3.4796e-19],\n [ 3.2511e-18, 1.4864e-18, 3.1303e-19, ..., 1.0763e-18,\n 1.2831e-18, 8.0351e-19]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.7010e-11, 1.1751e-11, 2.4259e-11, ..., 4.5419e-11, 3.5358e-11,\n 4.9609e-11],\n [8.1240e-13, 4.3087e-13, 1.5575e-13, ..., 1.2898e-12, 4.7020e-13,\n 1.1477e-12],\n [5.3736e-14, 9.7083e-14, 1.0408e-13, ..., 1.7537e-13, 6.3192e-14,\n 3.1926e-14],\n ...,\n [4.6258e-12, 1.4004e-12, 3.1136e-12, ..., 2.6822e-12, 5.0024e-12,\n 6.3781e-12],\n [1.7000e-13, 2.2849e-13, 8.1618e-13, ..., 1.0648e-12, 1.7817e-12,\n 3.4012e-12],\n [1.0849e-11, 5.5358e-12, 7.7866e-12, ..., 9.3043e-12, 1.4749e-11,\n 2.0676e-11]], device='cuda:0')" + }, + "45": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-7.7497e-16, 8.7963e-17, 2.4268e-17, 1.4913e-17, 4.4136e-16,\n 1.5867e-16, -1.4429e-16, 8.3219e-17, -1.1609e-15, -6.7880e-17,\n -7.9789e-17, 1.6704e-16, -5.6555e-16, -6.8298e-16, -4.9847e-17,\n -3.6647e-16, -5.5285e-16, 1.4054e-16, -2.6610e-16, 1.6165e-16,\n 9.2038e-18, 7.9766e-16, 7.8969e-17, 4.7152e-16, -7.1377e-16,\n -1.0717e-17, 1.1996e-16, -6.4093e-16, -3.8332e-16, -2.1018e-16,\n 2.7465e-16, -1.3884e-16, 1.4055e-17, -2.1515e-18, -1.0240e-16,\n 3.1338e-16, -1.8082e-16, 1.8918e-16, 6.4953e-16, -9.9665e-17,\n 3.5492e-16, 1.3583e-16, 4.9000e-16, 1.3907e-16, 1.9695e-16,\n 2.9377e-16, -4.5083e-16, -2.7292e-16, -1.6803e-16, 6.8103e-17,\n -7.6598e-17, 5.6444e-16, -1.2659e-16, -2.1911e-16, -6.6885e-16,\n 1.7654e-16, 3.4353e-16, 4.1134e-16, -1.6956e-17, 3.7310e-17,\n 1.4622e-16, -1.0120e-16, -1.4911e-15, -3.4362e-16, -6.4117e-16,\n 2.6198e-16, 5.9801e-19, -5.5738e-17, 1.1880e-18, 1.5729e-17,\n -5.3397e-16, 3.8561e-16, 2.2780e-16, -2.8020e-17, -1.1105e-15,\n 2.0661e-19, -4.5435e-16, -2.6820e-16, 7.1115e-16, -7.8788e-16,\n -3.7123e-17, 3.0883e-17, 4.5882e-17, 5.7235e-17, 5.9106e-16,\n 2.8014e-16, -1.4783e-17, -5.8397e-17, 1.0368e-15, -1.2267e-17,\n -5.1885e-17, 5.5796e-17, -3.7201e-17, 8.7077e-16, -2.5780e-16,\n -6.5217e-17, -3.2840e-16, -1.3015e-16, 1.4992e-16, -6.2622e-16,\n 2.1291e-16, 6.8891e-16, -9.7153e-16, -1.1804e-17, 4.3792e-16,\n 3.4845e-16, -8.3173e-16, 3.8768e-16, -6.0751e-17, 7.5484e-16,\n -6.6843e-16, 7.9243e-16, 3.2443e-18, 2.0744e-17, -1.8546e-15,\n -1.4416e-16, 1.8443e-17, -3.5899e-16, 2.4447e-16, -6.2111e-16,\n 9.6819e-17, 9.5760e-17, 4.9968e-17, 1.2720e-15, -2.6153e-16,\n -5.1348e-16, -2.4430e-16, 2.0325e-16, 1.7229e-16, 4.0193e-16,\n 1.9094e-16, -1.3676e-16, -1.4233e-16, 7.4510e-17, -1.5649e-16,\n -1.2135e-16, 1.2820e-17, 3.8424e-16, 3.6655e-16, -2.8471e-16,\n 2.4732e-16, -3.7429e-16, 3.0102e-16, -7.2157e-16, -4.8855e-16,\n -7.7530e-17, -1.2359e-16, -5.6261e-16, 6.5607e-16, -1.2540e-16,\n -1.8732e-16, -3.3660e-16, -3.5680e-17, 6.2108e-16, 3.0043e-16,\n -1.2604e-16, -1.2886e-16, -2.3192e-16, 1.9779e-16, -9.9247e-17,\n 3.8868e-16, -8.9877e-16, -1.6321e-16, -6.2470e-16, -1.7805e-16,\n 4.7458e-16, -5.1225e-16, -1.4553e-16, -4.5228e-16, 2.3724e-16,\n 4.5142e-16, 4.3512e-17, -1.2726e-16, -1.7001e-16, -4.0933e-17,\n 5.4033e-17, -4.2794e-17, -6.1401e-17, -3.4335e-16, 2.0689e-16,\n -2.9497e-17, 3.5311e-16, 5.9482e-16, -5.6735e-17, -2.1242e-16,\n 1.6656e-17, 9.0730e-17, 3.0835e-16, 1.8477e-16, -4.6312e-18,\n 6.1082e-16, 8.4508e-17, -1.6296e-16, -1.2352e-15, 4.7891e-16,\n 4.6498e-16, 6.3202e-18, 1.5147e-16, 1.4563e-16, 6.2529e-16,\n 4.2083e-16, -7.9300e-16, -1.8001e-16, 1.4471e-16, 2.6057e-16,\n -3.1023e-17, -5.3436e-16, 5.5828e-16, -3.9278e-17, -1.5029e-16,\n 2.7370e-16, -6.2203e-17, 8.3048e-17, 1.9498e-16, 1.8716e-16,\n -5.8471e-17, -1.2388e-16, 4.6340e-16, 7.9534e-16, 2.0241e-16,\n 5.7077e-16, 3.7213e-16, -1.8130e-16, -4.7069e-16, -1.2397e-16,\n -5.6821e-17, -1.9234e-16, -5.3327e-16, 1.8518e-18, 9.1679e-17,\n -9.3460e-17, 1.0095e-16, 7.0337e-16, -1.6264e-17, 3.9554e-16,\n -8.4370e-17, -5.3184e-17, 5.3441e-16, -2.2706e-17, -4.1620e-17,\n -9.3816e-17, 2.7952e-16, 3.8868e-16, 1.6908e-16, 2.9043e-16,\n 2.8640e-16, -5.2362e-16, 4.3912e-16, -4.9774e-16, -4.6204e-17,\n 4.2646e-16, 5.6957e-16, -3.7207e-17, -5.8286e-17, 2.1289e-16,\n 9.7070e-16], device='cuda:0')", + "exp_avg_sq": "tensor([8.2357e-06, 2.1136e-07, 6.3588e-09, 2.3228e-05, 1.5282e-05, 3.2702e-07,\n 2.3808e-08, 2.0573e-06, 2.2071e-05, 3.3544e-06, 3.4012e-06, 2.4930e-05,\n 2.3605e-05, 2.9525e-07, 7.8785e-08, 9.4848e-08, 1.1223e-07, 1.3800e-07,\n 1.8985e-07, 3.0212e-06, 1.4952e-08, 2.1573e-06, 4.8028e-07, 7.3271e-07,\n 6.3919e-07, 2.0492e-07, 9.4236e-07, 2.6627e-05, 3.8986e-06, 5.5759e-08,\n 7.0297e-05, 2.4263e-08, 1.6326e-05, 5.7635e-07, 3.5989e-06, 4.7481e-05,\n 3.6374e-07, 5.0755e-07, 7.4352e-06, 1.4140e-08, 2.8067e-06, 5.7217e-06,\n 9.9291e-08, 7.8859e-07, 5.7326e-08, 3.7827e-08, 1.2005e-07, 1.5838e-05,\n 2.4102e-08, 4.7812e-06, 3.0894e-06, 2.5766e-06, 2.3453e-06, 4.3898e-07,\n 5.1825e-07, 2.0980e-06, 4.0329e-08, 3.8213e-07, 3.8024e-05, 5.9455e-07,\n 7.6567e-06, 4.3146e-06, 1.4569e-05, 5.0795e-05, 2.2534e-05, 2.2283e-05,\n 1.4313e-09, 1.6020e-08, 1.0889e-07, 6.2169e-06, 4.3857e-05, 4.5228e-06,\n 1.1978e-06, 5.8576e-07, 6.4006e-06, 2.3256e-07, 1.8415e-07, 5.7722e-07,\n 1.7789e-05, 2.6584e-05, 7.2338e-09, 5.0713e-08, 1.8075e-06, 6.6482e-06,\n 1.6085e-05, 7.0852e-07, 2.2596e-09, 7.0415e-09, 3.1775e-07, 6.1020e-06,\n 1.1714e-05, 1.5093e-06, 8.8076e-06, 1.6995e-05, 3.8935e-08, 2.2942e-08,\n 3.8901e-07, 8.4917e-06, 4.3778e-06, 1.9058e-06, 5.0260e-06, 2.8306e-05,\n 1.7430e-05, 6.6965e-07, 6.2490e-06, 6.4998e-06, 9.5531e-07, 9.0103e-07,\n 2.6537e-08, 2.3380e-05, 7.3855e-06, 4.3453e-07, 3.1166e-06, 4.4385e-07,\n 1.1289e-05, 3.5238e-07, 7.7069e-09, 2.0669e-07, 2.4803e-08, 1.6906e-07,\n 1.5811e-08, 5.4195e-07, 6.5985e-08, 2.5398e-05, 8.6583e-08, 6.5718e-06,\n 2.3535e-06, 1.5074e-05, 1.5315e-06, 7.6976e-05, 2.0222e-06, 1.8829e-08,\n 2.1745e-06, 1.2432e-07, 1.2187e-07, 1.6726e-05, 3.6346e-08, 4.2727e-07,\n 5.5539e-07, 2.0007e-08, 6.3543e-06, 4.5248e-07, 2.2915e-08, 1.0540e-06,\n 2.7295e-05, 9.2162e-09, 5.9927e-07, 1.0213e-06, 1.4546e-05, 4.3371e-07,\n 3.4324e-05, 2.8482e-07, 1.5082e-06, 1.6238e-07, 3.6682e-06, 2.6976e-06,\n 1.3241e-07, 7.2174e-06, 4.9269e-05, 4.3122e-07, 4.2739e-06, 6.1468e-06,\n 8.7370e-06, 3.6000e-05, 1.9925e-06, 1.1951e-05, 1.0190e-05, 1.3421e-07,\n 1.2991e-05, 1.1934e-05, 2.7186e-05, 5.4121e-08, 1.0576e-06, 8.1882e-07,\n 2.0266e-07, 1.5534e-06, 1.2298e-08, 2.9524e-07, 1.2691e-06, 8.4763e-07,\n 1.3257e-05, 1.8511e-05, 5.5011e-07, 7.7150e-08, 4.1281e-07, 5.0952e-05,\n 2.7740e-07, 1.7002e-07, 1.3729e-05, 9.5775e-06, 1.3795e-05, 9.7108e-08,\n 1.4811e-08, 2.3292e-05, 1.2974e-06, 4.7059e-06, 4.6067e-07, 5.2678e-06,\n 2.7242e-05, 3.7571e-06, 4.6085e-08, 1.4411e-04, 9.3878e-08, 1.7941e-05,\n 1.2953e-05, 5.4704e-06, 1.0250e-05, 6.1050e-06, 4.7610e-07, 2.2045e-07,\n 2.0308e-05, 8.3277e-09, 4.9598e-06, 9.4006e-09, 1.3701e-06, 1.2808e-07,\n 6.2692e-08, 9.3753e-07, 1.5981e-05, 8.6455e-08, 3.5034e-05, 7.2153e-07,\n 1.1475e-07, 4.1111e-05, 1.0872e-08, 4.8190e-07, 4.9208e-06, 2.2322e-06,\n 3.6855e-07, 3.4736e-07, 3.5168e-08, 2.9168e-06, 8.0853e-07, 1.7647e-07,\n 1.2979e-05, 1.2926e-06, 1.4850e-08, 8.0921e-05, 5.8595e-08, 6.5957e-08,\n 4.6708e-06, 8.3537e-05, 7.3022e-08, 5.6483e-06, 1.0934e-05, 7.2877e-08,\n 3.2266e-05, 5.4655e-06, 1.0524e-05, 3.9422e-09, 1.1230e-06, 4.1966e-05,\n 3.5723e-07, 1.2985e-06, 1.9687e-07, 3.8583e-06], device='cuda:0')" + }, + "46": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.5178e-17, -5.1984e-18, 1.3706e-19, -5.4501e-17, -4.1817e-17,\n -1.9087e-20, -1.5007e-18, -5.7621e-18, -8.6822e-17, -3.2349e-17,\n -4.4070e-17, -3.6965e-20, -1.3209e-16, -4.2606e-17, 1.0525e-18,\n -8.1899e-18, -5.9043e-18, 2.3369e-20, 2.8190e-18, -5.4605e-19,\n -8.9709e-18, 2.8136e-19, -4.3798e-18, -6.7564e-18, -4.5783e-17,\n -1.4320e-17, 6.9180e-19, -1.1346e-16, -8.5812e-17, -9.8631e-19,\n -4.1120e-18, -5.5338e-18, -2.8691e-17, 8.1770e-20, 9.7859e-19,\n -4.9295e-18, -4.2203e-19, 5.2738e-19, -5.0699e-17, 1.8053e-19,\n -1.6145e-18, -8.4284e-17, -6.5180e-19, -1.0678e-19, 3.9411e-19,\n -1.5843e-18, -1.6652e-17, -5.2777e-17, -2.8476e-19, -3.6445e-17,\n -4.0794e-17, -1.4928e-19, -1.5622e-17, -1.2168e-18, -3.1956e-17,\n -7.9055e-18, -4.1895e-19, -3.7382e-18, -7.0751e-17, -7.3722e-18,\n 6.6309e-19, 1.3545e-18, -6.4180e-17, -6.1563e-17, -7.7560e-17,\n 2.8839e-19, -1.2841e-18, 5.1496e-19, -7.1685e-19, 3.1810e-19,\n -1.0475e-16, -8.9576e-19, 1.3660e-19, -1.0142e-18, -5.2374e-17,\n -3.6267e-17, -1.0622e-17, 1.3019e-18, -2.1319e-18, -7.1643e-17,\n -2.4729e-18, -5.0033e-19, 7.6638e-19, -9.0942e-18, -7.4927e-17,\n -3.8999e-20, -9.1097e-19, 5.4188e-19, -2.1644e-18, -7.5017e-17,\n -1.8924e-17, 5.2166e-19, -3.5824e-17, 6.5849e-19, -4.7823e-18,\n 8.6198e-19, 2.8506e-18, -1.1909e-16, -1.1252e-17, -3.9257e-17,\n -1.6894e-17, -6.2135e-17, -1.0606e-16, -1.9001e-18, -4.9286e-20,\n -2.3807e-17, -3.5778e-17, 1.7419e-19, -2.5685e-18, 1.3164e-20,\n -3.9487e-17, -1.5306e-18, 5.4690e-19, -3.9255e-19, -1.0741e-16,\n -7.9483e-18, 4.6412e-20, 4.8871e-18, 2.9202e-20, -1.3908e-17,\n 1.7790e-19, 2.0254e-19, -3.0347e-20, -4.0319e-17, -1.0519e-19,\n -7.5709e-17, 1.9367e-18, 2.6808e-19, 3.7165e-20, -1.4206e-18,\n -8.4111e-20, -6.7684e-18, -2.1372e-18, 2.6728e-19, -3.6234e-18,\n -1.1288e-16, -6.0410e-19, -1.3414e-18, -9.0948e-19, 3.2936e-18,\n 4.1814e-19, -5.1133e-18, -2.2416e-19, -8.3313e-17, -1.2504e-17,\n -2.2028e-18, -9.0476e-18, -2.0517e-17, -1.3587e-19, -5.2740e-19,\n -1.1183e-17, -8.8539e-18, -7.2322e-19, -1.9788e-18, -9.7272e-17,\n 3.0278e-19, -1.3067e-17, -2.3043e-17, -2.1691e-19, -5.1791e-19,\n -8.2646e-18, -4.1623e-17, -3.9314e-17, -1.1918e-16, -1.1946e-17,\n -9.7135e-21, -1.1972e-16, -1.8389e-17, -5.3121e-17, -3.0951e-19,\n -5.0560e-17, -3.0245e-20, -8.0391e-19, -1.7715e-18, -1.7269e-18,\n -2.5732e-20, -2.7883e-18, -3.4380e-18, 4.1486e-18, -1.9648e-18,\n -1.5852e-17, -9.0361e-18, -1.2675e-18, -3.1310e-19, 7.1831e-19,\n -1.2092e-17, 6.6038e-20, -1.1974e-18, 5.1172e-20, -9.5751e-19,\n -1.0119e-17, -1.9345e-19, 2.7210e-19, -8.8342e-17, -3.7737e-19,\n -4.6779e-17, -3.1238e-18, -5.4688e-17, -8.2776e-17, -1.4446e-17,\n -5.7665e-19, -1.2025e-16, -2.7113e-18, -1.0603e-16, -5.9730e-17,\n 1.0294e-18, -4.7799e-17, -1.2238e-17, -1.0012e-18, -6.5037e-18,\n 3.1271e-19, -2.7016e-18, 3.5609e-19, 1.0470e-19, -3.5752e-20,\n -1.8839e-18, -9.3265e-19, -1.0739e-18, -1.2921e-18, 3.7091e-19,\n 1.2696e-19, -1.3259e-20, -1.1012e-18, -5.1945e-17, -2.0257e-18,\n -4.5332e-18, -4.7335e-17, -4.6412e-18, 1.2354e-18, -5.5590e-18,\n -1.8273e-18, -1.3515e-17, -9.3866e-19, -1.5794e-18, -7.8499e-18,\n -7.4031e-18, 6.5131e-19, -3.2442e-18, -9.6139e-19, -6.9866e-19,\n -2.0184e-18, -4.0912e-18, -1.2497e-18, -7.2386e-19, -2.7934e-19,\n 4.1391e-20, -8.9091e-17, -3.9590e-18, -1.1428e-16, -1.6985e-19,\n -1.3726e-19, -5.8912e-18, -4.0153e-17, 3.6215e-19, -2.1158e-18,\n -3.5756e-18], device='cuda:0')", + "exp_avg_sq": "tensor([4.5853e-09, 5.2168e-12, 4.5429e-12, 1.5650e-08, 1.3660e-08, 1.2952e-11,\n 3.9971e-13, 2.2952e-09, 1.9103e-08, 9.0220e-09, 1.3576e-08, 2.8444e-10,\n 7.8545e-08, 7.1658e-09, 1.1027e-12, 9.3501e-12, 5.6585e-12, 4.6568e-11,\n 1.1513e-13, 7.5875e-10, 2.2341e-10, 2.9747e-10, 2.3421e-09, 1.1583e-10,\n 2.6681e-09, 1.2434e-09, 1.0719e-10, 6.3342e-08, 1.4108e-08, 4.6065e-13,\n 2.2333e-08, 1.2284e-10, 3.1364e-09, 9.1651e-13, 4.0637e-12, 1.5297e-08,\n 2.3359e-13, 2.6743e-10, 2.0910e-08, 6.1361e-12, 1.4157e-09, 2.3688e-08,\n 6.4828e-11, 1.6434e-13, 3.9376e-11, 8.4986e-11, 9.9170e-13, 5.3012e-09,\n 7.0967e-13, 2.9871e-09, 2.2108e-09, 3.7328e-10, 9.0229e-12, 3.4359e-12,\n 1.1424e-10, 1.3826e-09, 2.4778e-12, 1.2619e-10, 3.7046e-08, 7.3769e-12,\n 4.2286e-10, 3.3059e-12, 1.7832e-09, 1.9864e-08, 1.7392e-08, 2.3897e-09,\n 2.9086e-12, 3.6513e-12, 6.0836e-13, 7.9227e-12, 5.2948e-08, 9.6662e-13,\n 6.0577e-11, 6.3216e-13, 7.8273e-10, 8.0623e-09, 1.8869e-12, 9.5607e-13,\n 7.7615e-10, 9.8458e-09, 1.4187e-12, 1.0439e-10, 7.2038e-11, 2.8342e-09,\n 5.3820e-08, 8.1138e-11, 3.2929e-12, 5.5268e-12, 8.4201e-11, 1.9121e-08,\n 8.3824e-11, 1.3630e-11, 4.8105e-09, 8.9522e-10, 1.3486e-10, 3.6369e-11,\n 2.7278e-12, 5.1051e-08, 7.1502e-09, 4.0800e-10, 2.0932e-10, 5.7186e-08,\n 2.5463e-08, 1.1232e-10, 2.4731e-10, 2.0089e-09, 8.0238e-11, 3.6057e-12,\n 7.4694e-13, 2.7506e-09, 6.4105e-10, 1.0479e-10, 5.3778e-11, 5.1935e-11,\n 9.5180e-09, 5.3353e-11, 7.9517e-11, 1.8871e-11, 1.3482e-12, 1.0307e-11,\n 1.8451e-11, 3.4889e-11, 1.5586e-12, 6.5269e-08, 1.1766e-13, 1.2350e-08,\n 7.6232e-12, 3.6840e-10, 1.0746e-10, 2.1709e-08, 4.6486e-14, 9.9144e-11,\n 1.0240e-09, 4.8734e-11, 1.6997e-12, 6.1324e-08, 3.0698e-12, 2.3649e-12,\n 3.8146e-11, 1.0688e-12, 3.1572e-10, 8.0737e-10, 5.0781e-12, 1.5337e-08,\n 1.4118e-08, 1.8163e-12, 9.7226e-10, 3.4246e-12, 2.5367e-09, 2.5826e-13,\n 1.4867e-08, 5.9007e-10, 4.8421e-13, 1.5630e-10, 3.5814e-08, 6.3088e-12,\n 4.8040e-10, 1.2454e-09, 8.8774e-10, 2.0941e-10, 5.6238e-09, 7.3314e-10,\n 2.9826e-09, 7.1325e-08, 2.1767e-11, 2.9704e-10, 3.5464e-08, 3.6512e-09,\n 4.9967e-09, 2.0439e-09, 2.6424e-08, 8.9877e-11, 3.8672e-13, 3.5881e-12,\n 1.4033e-10, 2.0916e-12, 6.9627e-11, 2.8075e-10, 9.9473e-13, 9.9242e-12,\n 1.1217e-08, 8.0423e-09, 4.0794e-11, 2.0573e-12, 4.3947e-13, 2.8950e-08,\n 2.2484e-11, 5.4840e-10, 9.3547e-10, 5.7838e-12, 8.4497e-10, 1.1234e-12,\n 3.8730e-12, 1.1570e-08, 4.3402e-10, 9.3663e-09, 2.1070e-10, 9.6674e-09,\n 6.9617e-08, 2.0727e-09, 3.6922e-12, 1.3757e-07, 2.6096e-13, 7.2086e-08,\n 1.8932e-08, 4.5251e-12, 3.3213e-09, 1.4753e-09, 1.2640e-10, 4.6442e-10,\n 1.2543e-09, 1.8841e-12, 5.0422e-11, 3.7842e-12, 4.1440e-11, 1.6971e-10,\n 2.7222e-13, 1.3650e-10, 6.4144e-10, 1.3243e-12, 5.2522e-09, 2.6265e-10,\n 2.0499e-13, 8.7803e-09, 2.8544e-12, 3.1813e-12, 2.6496e-09, 5.7883e-10,\n 3.6112e-12, 1.0230e-09, 1.7945e-12, 1.2990e-10, 2.9335e-10, 2.1367e-10,\n 2.9431e-09, 5.3588e-10, 5.1227e-12, 3.6591e-08, 6.7461e-11, 4.0595e-13,\n 1.5418e-09, 2.8364e-08, 6.2022e-11, 1.5311e-09, 2.7143e-10, 4.0130e-11,\n 3.2719e-08, 2.2226e-10, 3.7963e-08, 1.3226e-12, 2.0227e-10, 1.6949e-08,\n 7.9719e-09, 9.9296e-12, 7.2177e-12, 2.0333e-09], device='cuda:0')" + }, + "47": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-3.7528e-17, -1.6361e-17, 1.0414e-19, -3.9240e-17, -2.9734e-17,\n 1.0466e-20, 1.2058e-18, -1.6945e-17, -6.5663e-17, -3.3785e-17,\n -3.7608e-17, 3.2559e-19, -6.4478e-17, -4.6449e-17, -6.1725e-18,\n -2.6986e-17, -2.8114e-17, -9.6939e-19, -2.0779e-18, -9.1998e-18,\n 7.0846e-18, 8.4231e-19, -1.5938e-17, -1.2735e-17, -4.8135e-17,\n 1.0106e-17, 1.6296e-18, -6.2548e-17, -5.2894e-17, 8.0244e-19,\n -1.1856e-17, 4.2686e-18, -3.0849e-17, 6.3306e-20, -5.5596e-19,\n -1.2884e-17, 2.6875e-19, -3.7593e-18, -2.9906e-17, -1.3990e-19,\n -6.8970e-18, -4.5353e-17, 3.1769e-18, 2.8174e-19, -2.3968e-18,\n 4.4209e-18, -3.3382e-17, -4.2615e-17, 2.5019e-19, -3.2705e-17,\n -3.6517e-17, 3.9237e-18, -2.6842e-17, 9.9146e-19, -4.3011e-17,\n -1.7482e-17, 2.4592e-18, -1.0032e-17, -4.4343e-17, -1.8869e-17,\n -3.4099e-18, -8.5701e-19, -6.6522e-17, -4.6217e-17, -5.5054e-17,\n -3.6319e-18, 7.1097e-19, -3.5825e-19, -3.7424e-19, 6.7759e-20,\n -5.9183e-17, 4.1733e-18, -6.0660e-18, 4.7009e-19, -5.6950e-17,\n -3.4149e-17, -2.9949e-17, -9.9909e-19, -4.6495e-18, -5.6088e-17,\n 2.0908e-18, -4.8824e-19, -4.9991e-18, -2.0520e-17, -3.7609e-17,\n -5.2820e-19, 2.7881e-19, -3.2856e-19, 1.6874e-17, -4.5003e-17,\n -2.7417e-17, -3.2738e-18, -3.4479e-17, 1.4771e-18, 2.7320e-18,\n -8.8137e-18, -2.3598e-18, -5.6363e-17, -2.0320e-17, -4.4823e-17,\n -2.2030e-17, -3.2830e-17, -6.6234e-17, 9.4398e-19, 1.1976e-18,\n -2.4188e-17, -4.7464e-17, -1.6365e-18, -1.0025e-18, 1.5547e-18,\n -4.5370e-17, 1.1351e-17, -3.9220e-18, -1.4996e-19, -8.1447e-17,\n -2.3526e-17, -1.8599e-18, -4.0382e-18, 4.2425e-19, -3.5142e-17,\n -1.2053e-18, -6.6739e-19, -2.0148e-19, -2.0616e-17, 8.7903e-20,\n -5.2149e-17, -1.5469e-18, -6.8369e-19, 1.0846e-18, -7.3999e-18,\n 1.1331e-18, 5.0868e-18, -1.8418e-17, -5.3211e-18, 2.8182e-18,\n -5.4653e-17, -1.3210e-18, 4.7782e-18, 4.8271e-18, -2.6622e-18,\n -2.0255e-18, 3.4210e-18, 1.2863e-18, -5.7505e-17, -3.0841e-17,\n 1.0624e-18, 6.5278e-18, -3.7428e-17, -1.1880e-18, 7.2586e-19,\n -2.5870e-17, 5.7763e-18, 4.0673e-19, 8.9934e-18, -4.5945e-17,\n -2.3500e-19, 8.8860e-18, -3.2214e-17, 1.4320e-18, 3.7688e-19,\n -1.4460e-17, -5.0525e-17, -3.7057e-17, -6.3265e-17, -2.5773e-17,\n -1.8192e-18, -6.1889e-17, -2.9006e-17, -4.5962e-17, -7.4423e-18,\n -3.2294e-17, -1.2254e-18, 5.9418e-19, 1.2001e-18, 1.0655e-18,\n -4.5537e-19, 2.0163e-18, 2.8672e-18, -3.1778e-18, -1.0863e-17,\n -2.5645e-17, -1.5668e-17, 8.0789e-18, -1.3494e-17, -5.3701e-19,\n -2.2720e-17, -7.1580e-19, -7.9897e-18, -5.3976e-18, 3.4501e-19,\n -1.3382e-17, -2.3238e-19, -1.8304e-19, -6.7317e-17, 5.7646e-18,\n -3.0918e-17, 2.2153e-18, -3.7443e-17, -4.5031e-17, -1.5852e-17,\n 2.7348e-18, -6.6260e-17, 2.1602e-18, -4.9878e-17, -3.7634e-17,\n -3.6866e-19, -4.5621e-17, -1.5509e-17, 6.6646e-19, 4.8891e-18,\n -2.8324e-18, 2.1715e-18, -1.7677e-18, -5.4304e-19, -7.1661e-18,\n 1.1183e-18, 6.2566e-19, 5.7325e-18, 1.0622e-17, -2.0473e-18,\n 5.0434e-19, 1.4099e-18, 8.1759e-19, -4.5852e-17, 1.6476e-18,\n 3.5995e-18, -4.0282e-17, -2.7780e-17, -4.6454e-18, -1.7018e-17,\n 1.4269e-18, -2.2139e-17, 7.5797e-18, 1.1714e-19, -1.4032e-17,\n -2.1313e-17, -4.3429e-19, -7.9968e-18, 2.8761e-19, 4.8575e-19,\n -1.6488e-17, -1.1895e-17, 4.4962e-18, -9.6866e-18, 3.0820e-18,\n -1.0513e-18, -5.5662e-17, -9.7999e-18, -6.0526e-17, -6.6430e-18,\n -1.8570e-18, -1.0278e-17, -3.5809e-17, -4.5384e-18, -1.1565e-17,\n -4.8680e-18], device='cuda:0')", + "exp_avg_sq": "tensor([5.0615e-09, 4.7721e-10, 2.4901e-12, 1.9853e-08, 1.3712e-08, 7.4065e-12,\n 1.2870e-12, 1.8813e-09, 2.2336e-08, 7.6964e-09, 8.6742e-09, 6.9981e-09,\n 2.9572e-08, 4.5020e-09, 2.2301e-10, 3.3041e-10, 2.4102e-10, 2.3915e-11,\n 4.9136e-12, 1.3261e-09, 1.2064e-10, 5.6153e-10, 2.3127e-09, 1.4136e-09,\n 4.2669e-09, 4.5193e-10, 5.4469e-11, 3.0153e-08, 1.0050e-08, 9.9528e-12,\n 2.0459e-08, 7.1115e-11, 1.0880e-08, 1.9796e-11, 7.5608e-10, 9.4693e-09,\n 2.8594e-11, 1.2991e-10, 9.4433e-09, 1.7696e-12, 1.7334e-09, 1.3205e-08,\n 3.3176e-11, 3.9538e-11, 2.1115e-11, 2.2664e-11, 9.6452e-10, 1.5035e-08,\n 7.9610e-14, 7.4408e-09, 5.0264e-09, 1.7004e-09, 2.5478e-09, 1.1737e-10,\n 1.7243e-09, 1.1603e-09, 1.0700e-12, 1.1350e-09, 2.8169e-08, 9.7076e-10,\n 1.9699e-09, 8.1660e-10, 1.2086e-08, 3.4559e-08, 2.0928e-08, 6.0436e-09,\n 1.0099e-12, 4.8970e-12, 1.6125e-10, 1.4443e-09, 3.5200e-08, 4.1261e-10,\n 3.4357e-10, 9.6586e-11, 6.5607e-09, 4.3782e-09, 6.6356e-10, 3.4864e-11,\n 5.9406e-10, 2.0402e-08, 1.2979e-12, 5.7400e-11, 9.6315e-10, 1.6465e-09,\n 2.1139e-08, 3.6402e-11, 1.1350e-12, 7.3563e-13, 4.4385e-11, 1.1755e-08,\n 7.4621e-09, 8.5358e-10, 1.1146e-08, 8.6518e-10, 4.5985e-11, 1.7249e-11,\n 9.4576e-11, 1.7135e-08, 4.2242e-09, 3.7031e-09, 4.1004e-09, 2.0722e-08,\n 1.9497e-08, 4.4158e-11, 7.8444e-10, 6.6014e-09, 2.3176e-09, 2.9169e-10,\n 9.1554e-11, 2.8617e-09, 6.4333e-09, 5.5986e-11, 4.9428e-10, 2.6037e-11,\n 1.3963e-08, 1.5848e-09, 4.2645e-11, 4.8717e-11, 4.9449e-13, 6.4016e-10,\n 9.4338e-12, 1.5285e-11, 9.1614e-11, 2.3668e-08, 8.8430e-13, 1.2029e-08,\n 4.4152e-10, 3.2596e-09, 4.5946e-11, 1.9924e-08, 4.3783e-11, 5.5055e-11,\n 2.2843e-09, 2.4237e-11, 3.6098e-11, 2.3565e-08, 2.5404e-10, 1.1760e-10,\n 2.1336e-11, 1.7556e-13, 1.2025e-09, 3.9992e-10, 2.6891e-12, 7.9038e-09,\n 9.1357e-09, 1.2507e-11, 3.4929e-10, 1.6228e-09, 2.6713e-09, 1.0407e-11,\n 9.2729e-09, 3.3284e-10, 8.0112e-11, 6.7906e-11, 1.2855e-08, 5.5662e-10,\n 2.4922e-10, 2.5200e-09, 1.3346e-08, 9.1359e-11, 3.7920e-09, 7.1238e-09,\n 7.9189e-09, 3.4514e-08, 1.5797e-09, 7.6160e-10, 1.6147e-08, 3.3834e-09,\n 1.0413e-08, 3.0671e-09, 2.2562e-08, 4.8904e-11, 9.3173e-11, 1.5612e-10,\n 7.3159e-11, 1.5819e-10, 3.5621e-11, 1.3640e-10, 1.4820e-10, 6.0344e-10,\n 5.7507e-09, 3.5550e-09, 2.2455e-11, 7.2210e-10, 2.0050e-11, 1.4109e-08,\n 1.1472e-11, 1.4025e-09, 2.0215e-09, 2.8280e-09, 4.9028e-10, 5.0055e-11,\n 1.0239e-12, 2.0484e-08, 2.0659e-10, 7.3627e-09, 8.8179e-11, 6.8824e-09,\n 3.1100e-08, 3.2634e-09, 1.7238e-12, 8.0908e-08, 7.4680e-12, 2.3954e-08,\n 1.3594e-08, 1.0367e-09, 1.0646e-08, 7.0946e-10, 5.6995e-11, 2.2545e-10,\n 3.8981e-09, 5.9815e-13, 6.2720e-10, 1.9737e-12, 3.0497e-10, 8.6763e-11,\n 8.5261e-12, 6.8414e-11, 3.0488e-09, 8.0073e-13, 8.1349e-09, 1.2487e-10,\n 1.1018e-11, 2.6097e-08, 2.3971e-13, 1.6487e-10, 6.7669e-09, 2.6489e-09,\n 6.4369e-10, 2.3793e-09, 8.8520e-12, 2.3073e-09, 1.5046e-10, 9.5799e-11,\n 1.4772e-09, 9.5638e-10, 4.5780e-12, 2.2349e-08, 3.9029e-11, 2.9014e-11,\n 2.2878e-09, 2.1788e-08, 2.7231e-11, 2.4220e-09, 2.3587e-09, 2.0221e-11,\n 2.5472e-08, 3.7339e-10, 1.8331e-08, 1.1057e-10, 5.7361e-10, 7.8125e-09,\n 5.3917e-09, 1.7242e-10, 3.7258e-10, 1.0582e-09], device='cuda:0')" + }, + "48": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.6311e-19, -2.4941e-19, -3.5883e-20, ..., 1.4979e-18,\n -5.6796e-21, -1.1887e-19],\n [ 8.9492e-19, 1.1532e-19, 3.6216e-19, ..., -1.4139e-18,\n 9.9789e-20, 4.2605e-19],\n [ 7.4416e-19, 3.0507e-19, 9.8931e-20, ..., -1.4147e-19,\n 2.3051e-19, 2.4634e-19],\n ...,\n [ 3.9321e-19, 2.6139e-19, -3.6361e-19, ..., 3.2644e-19,\n 1.9319e-19, 2.4534e-19],\n [ 1.0080e-18, 6.1087e-19, 1.7711e-19, ..., 5.3959e-18,\n 2.4449e-19, 3.3481e-19],\n [-3.5469e-19, 4.6419e-20, -1.3839e-19, ..., -2.2933e-19,\n -4.3728e-20, -1.0336e-19]], device='cuda:0')", + "exp_avg_sq": "tensor([[1.1249e-11, 3.0342e-12, 7.1206e-12, ..., 7.2837e-12, 7.9762e-12,\n 1.1280e-11],\n [1.8320e-10, 5.3740e-11, 1.3748e-10, ..., 8.2748e-11, 1.9383e-10,\n 2.2108e-10],\n [9.6587e-12, 1.8198e-12, 4.3516e-12, ..., 5.9608e-12, 7.2564e-12,\n 1.2823e-11],\n ...,\n [4.8430e-12, 2.5236e-12, 3.8094e-12, ..., 1.8663e-12, 6.2089e-12,\n 3.2047e-12],\n [2.2163e-10, 6.5300e-11, 1.4278e-10, ..., 1.3641e-10, 2.1405e-10,\n 2.4966e-10],\n [3.4339e-12, 1.8761e-12, 3.1988e-12, ..., 1.9092e-12, 7.2077e-12,\n 6.6696e-12]], device='cuda:0')" + }, + "49": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.5265e-16, 2.1413e-16, 1.9196e-16, -4.8164e-16, 1.5509e-16,\n 2.2115e-16, 7.1934e-17, 1.1384e-16, -1.1152e-15, 9.9861e-17,\n 9.5429e-17, 3.3556e-16, -3.8452e-16, -4.6498e-17, 4.7276e-16,\n -9.8647e-16, -1.0816e-15, 2.2141e-17, -2.9739e-17, 9.2577e-17,\n -1.1460e-16, -7.2271e-17, 1.1189e-16, 2.6309e-16, 1.0740e-16,\n -1.5461e-15, 1.2866e-17, -1.2207e-15, -9.7159e-16, -1.0755e-16,\n -1.5323e-17, -4.1979e-16, -7.6905e-17, 1.6340e-16, 1.2117e-16,\n -3.0211e-17, 1.4763e-16, -3.4821e-16, 4.7553e-17, 2.8471e-16,\n 1.8572e-16, 6.0128e-18, 6.1320e-16, 2.3422e-17, -9.4966e-17,\n 5.9404e-16, -6.7600e-18, -1.1180e-15, 1.1201e-16, -4.5767e-16,\n 2.5311e-17, -2.7497e-16, 4.3231e-16, 4.7907e-16, 2.4838e-16,\n 1.0824e-15, 5.5277e-17, -5.2873e-17, -1.8615e-17, -6.3100e-18,\n 2.2062e-17, -1.6049e-16, -1.0636e-15, -5.1606e-16, 2.8127e-16,\n 4.3484e-16, -5.9638e-16, 4.4698e-16, -7.7946e-17, 6.6147e-17,\n -4.4445e-16, -1.1110e-16, 3.7513e-16, 5.0777e-16, -3.9706e-17,\n 6.9275e-16, 3.4590e-16, -1.9152e-16, 1.8295e-16, -7.0029e-16,\n 3.4651e-16, 1.5177e-15, 6.2430e-17, 4.8048e-16, -5.8416e-16,\n 5.2745e-16, 3.6487e-16, 1.6985e-16, 4.4372e-16, -1.0015e-16,\n -2.0086e-16, -1.3956e-16, -6.2143e-16, -1.5350e-16, -1.5397e-16,\n -4.7727e-16, 2.5529e-16, -3.0256e-16, 4.2420e-16, -4.9745e-18,\n 9.0155e-16, 2.0349e-16, -3.4118e-16, -1.2184e-16, 1.9885e-16,\n -1.5473e-16, -2.8961e-16, -2.0730e-16, -1.1741e-15, 2.0008e-16,\n -1.3487e-16, 6.8429e-16, 1.3781e-16, 3.3594e-16, -1.4049e-16,\n -1.9326e-16, -6.5414e-16, 1.1589e-16, 3.3881e-17, -1.0580e-15,\n 1.1663e-16, -1.5209e-16, 6.9115e-17, -1.6302e-17, -2.0194e-16,\n -2.2123e-16, 4.9606e-16, 2.4900e-16, 1.5053e-16, 4.7534e-16,\n 1.0781e-16, -9.9053e-17, 4.1133e-16, -3.8478e-16, -2.3310e-16,\n -1.3887e-17, 6.5144e-17, 9.1312e-17, 1.8752e-16, -9.4689e-18,\n 3.6814e-16, -3.1576e-16, 2.5115e-16, -1.0328e-16, 1.0443e-16,\n 3.9714e-16, 2.6548e-16, -9.1713e-16, 2.4385e-16, 2.5422e-16,\n -2.5620e-16, -7.6717e-16, 6.9473e-16, 1.9002e-16, -1.1192e-16,\n -2.0009e-16, -6.5600e-16, 3.4677e-17, 2.3961e-16, 1.8753e-16,\n -1.8908e-17, -1.9316e-17, 6.9988e-16, -4.3146e-17, 2.0082e-16,\n -3.4399e-17, -1.1515e-16, 3.4103e-17, 8.3151e-17, -1.4512e-16,\n -1.3070e-16, -1.0142e-16, 7.4606e-16, 4.1225e-16, 1.8700e-17,\n 4.9808e-16, -3.4652e-17, 2.3021e-17, -5.3813e-18, 7.7253e-16,\n -1.7473e-17, 1.1472e-16, -3.8818e-16, 2.2483e-17, -2.7041e-16,\n 1.1682e-17, 6.7746e-17, -5.7635e-17, 3.7280e-16, 7.7077e-16,\n 6.4152e-17, 6.4241e-16, 4.0189e-17, -1.0392e-15, -1.8250e-16,\n 5.0625e-17, -2.7533e-17, -8.5657e-17, -9.2378e-16, -6.8873e-17,\n 1.5292e-16, -3.2880e-16, -3.8606e-17, -8.9936e-16, -2.2081e-17,\n -2.8014e-16, -1.3002e-15, -1.2801e-16, 4.9749e-17, 2.2988e-16,\n 1.6841e-16, 4.0597e-17, 4.1042e-16, 1.3783e-16, 1.6195e-16,\n -2.9922e-16, 8.1315e-16, -1.5897e-16, 4.8959e-16, 6.2085e-16,\n 1.5576e-16, 3.7525e-16, -2.1404e-16, 6.0044e-17, 7.7525e-17,\n -2.5568e-16, -8.1457e-17, -9.2332e-16, -5.5897e-16, -9.0300e-18,\n 4.8547e-17, -5.6449e-17, 4.4035e-16, -1.6540e-16, 1.3090e-16,\n -3.4515e-17, -3.8708e-17, 4.1212e-17, -1.8833e-17, 1.2396e-15,\n 3.0112e-17, 2.4321e-16, -3.6802e-16, -1.3822e-16, -1.4161e-16,\n 7.5259e-17, -9.6858e-17, 2.8110e-16, -1.3382e-17, 7.1064e-16,\n 4.7319e-16, -7.5004e-17, 3.6875e-17, 1.5639e-16, 2.8761e-16,\n -4.4181e-17], device='cuda:0')", + "exp_avg_sq": "tensor([2.4776e-06, 4.1547e-05, 1.8848e-06, 2.7809e-05, 4.2811e-06, 1.4862e-07,\n 1.8828e-07, 2.7934e-06, 1.4746e-05, 6.4354e-06, 8.0617e-07, 1.0291e-07,\n 2.7840e-06, 3.6156e-07, 1.5796e-06, 2.0670e-05, 4.3634e-05, 2.9495e-07,\n 1.2051e-08, 4.4669e-07, 3.7406e-06, 4.8255e-07, 5.2982e-07, 9.4592e-07,\n 2.5668e-07, 1.7373e-06, 9.0780e-07, 6.5094e-05, 1.6613e-06, 8.9285e-07,\n 7.8101e-06, 1.6682e-05, 9.8037e-07, 3.8839e-07, 4.4970e-06, 2.6299e-06,\n 2.0363e-06, 2.6820e-05, 1.3537e-07, 1.0360e-05, 9.2681e-07, 1.8720e-07,\n 2.2360e-07, 1.7767e-06, 5.2173e-07, 1.9942e-07, 8.8150e-08, 1.1484e-05,\n 2.3810e-06, 2.4011e-06, 5.3566e-08, 6.6790e-08, 6.1590e-05, 5.7150e-05,\n 2.9632e-06, 6.9367e-06, 8.4283e-09, 2.2898e-06, 1.2716e-06, 4.2164e-07,\n 5.0815e-06, 8.3185e-07, 3.5390e-05, 2.3857e-05, 2.2842e-05, 3.4781e-05,\n 5.1507e-07, 1.4766e-07, 1.1423e-08, 9.0139e-06, 2.9600e-05, 9.4743e-09,\n 2.1874e-06, 5.6242e-06, 4.6757e-07, 1.1154e-06, 8.1411e-07, 1.0903e-07,\n 5.1147e-06, 7.9336e-06, 2.7026e-06, 2.4508e-05, 2.5669e-06, 9.1439e-06,\n 1.5812e-07, 5.8744e-06, 2.5637e-06, 1.8231e-06, 1.1284e-05, 4.0143e-08,\n 3.2892e-07, 1.7484e-08, 6.0827e-06, 7.5769e-08, 1.2225e-07, 2.6043e-06,\n 1.2643e-07, 4.5214e-08, 1.1434e-06, 3.2657e-08, 2.1108e-05, 8.5191e-08,\n 4.6720e-09, 9.8442e-07, 8.6387e-06, 1.3981e-08, 1.8229e-06, 2.3259e-08,\n 9.4410e-06, 2.0111e-07, 1.4526e-08, 1.1456e-07, 5.7597e-06, 9.2072e-07,\n 1.0328e-05, 1.1277e-06, 9.3719e-06, 2.2868e-05, 5.0214e-07, 1.0267e-06,\n 6.7613e-06, 3.8330e-08, 1.6756e-07, 7.6572e-06, 6.6116e-08, 1.6882e-05,\n 3.2716e-05, 4.3320e-06, 5.4286e-06, 7.9088e-05, 1.5220e-06, 1.2819e-08,\n 4.2713e-06, 1.0135e-06, 8.7552e-06, 9.5627e-08, 4.4805e-08, 1.6614e-07,\n 1.0906e-06, 1.2999e-08, 1.0520e-07, 5.6399e-07, 2.6981e-08, 5.0970e-09,\n 4.3116e-05, 3.6341e-06, 3.9465e-05, 1.1198e-05, 1.2473e-05, 3.7705e-07,\n 1.7624e-05, 9.5027e-05, 4.4860e-06, 9.4857e-08, 4.7933e-06, 9.5563e-08,\n 4.6460e-05, 2.4787e-07, 3.8202e-05, 1.7503e-07, 2.5893e-06, 1.2165e-07,\n 7.7851e-06, 6.8233e-08, 2.5149e-07, 5.8684e-07, 6.7297e-07, 2.7558e-07,\n 3.2859e-07, 1.8063e-07, 4.6337e-08, 2.2238e-06, 2.3657e-05, 2.5050e-05,\n 1.8095e-08, 2.6836e-07, 1.4121e-05, 2.6305e-05, 3.1333e-09, 7.1010e-06,\n 1.5021e-05, 9.4655e-06, 7.1121e-07, 1.6320e-09, 1.0552e-07, 4.8372e-05,\n 7.5903e-06, 3.9770e-07, 2.1808e-05, 1.0081e-04, 4.6930e-07, 7.2957e-06,\n 1.7102e-06, 1.8484e-05, 3.2573e-06, 1.7197e-07, 2.5714e-07, 6.4624e-07,\n 1.4762e-05, 4.5429e-07, 5.3461e-09, 4.2708e-06, 6.9906e-08, 2.2660e-05,\n 8.8214e-09, 1.0808e-06, 2.8950e-06, 2.6073e-07, 2.4671e-07, 2.0609e-06,\n 5.4819e-07, 7.2171e-08, 1.2376e-05, 3.5092e-06, 1.2627e-06, 1.0171e-07,\n 2.4984e-06, 4.4718e-08, 2.1819e-07, 1.5543e-05, 1.3061e-05, 2.2312e-05,\n 7.5843e-07, 2.2050e-06, 3.3490e-07, 1.1235e-05, 2.9639e-07, 2.3792e-06,\n 2.7100e-06, 3.3589e-07, 4.6718e-06, 9.5275e-08, 4.1756e-07, 6.2689e-06,\n 3.1657e-06, 1.1408e-06, 5.8099e-06, 5.9508e-07, 8.0678e-06, 1.0188e-05,\n 3.4410e-06, 7.6253e-07, 5.5067e-07, 4.2657e-08, 7.2985e-08, 4.6238e-09,\n 7.1632e-08, 5.1020e-06, 3.6898e-07, 5.0735e-06, 1.2652e-06, 1.9150e-08,\n 7.4423e-08, 1.4574e-06, 4.5797e-05, 1.1477e-06], device='cuda:0')" + }, + "50": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 4.1033e-19, -5.5961e-17, 2.3486e-19, -6.3569e-17, -1.8708e-17,\n 6.1035e-20, -1.9838e-17, -1.5211e-18, -9.0313e-17, -9.7133e-17,\n 4.0113e-19, -5.5684e-19, -1.9073e-17, -4.3207e-18, -8.9626e-18,\n -8.0401e-17, -6.0822e-17, 5.8597e-19, 5.0749e-19, 2.8484e-19,\n -3.9593e-17, -2.3667e-17, -3.2188e-18, -1.2292e-17, -5.8127e-17,\n -2.3582e-17, 9.2819e-19, -1.5336e-16, -6.9552e-17, 7.4421e-19,\n -1.0826e-18, -9.2286e-17, -8.0066e-18, 1.0115e-19, 8.5083e-20,\n -7.4314e-19, -9.7339e-19, -6.1849e-18, 3.3880e-19, 1.7602e-20,\n -1.9013e-18, -5.7406e-19, -5.7714e-19, 1.8567e-19, -1.2605e-18,\n -1.4995e-18, -7.8930e-20, -1.3775e-16, 5.4208e-19, -4.1583e-17,\n 2.5106e-19, 4.4562e-19, -7.5235e-17, -1.4079e-19, -1.6134e-17,\n 2.5792e-18, 7.0347e-20, -3.4790e-17, -1.1605e-17, -5.5345e-19,\n -3.8138e-20, 6.0554e-19, -9.1243e-17, -5.4429e-17, -4.6404e-17,\n -2.7830e-18, -3.7110e-17, -1.5719e-19, -3.2220e-18, 3.5015e-19,\n -5.5158e-17, 1.1884e-18, -8.5441e-19, -2.0281e-18, -4.6804e-18,\n -7.0520e-18, -4.3737e-17, 1.1887e-18, 1.3841e-19, -3.2183e-17,\n -7.3672e-18, -1.9013e-17, 6.3811e-20, -1.5671e-18, -1.0721e-17,\n -3.9155e-18, -1.1783e-19, -3.3602e-20, -1.0879e-18, -7.2979e-18,\n -2.3827e-18, -2.6100e-18, -9.0569e-17, -5.5160e-19, -5.3252e-18,\n -2.5792e-17, -1.0673e-19, -9.8281e-18, -1.2456e-17, 6.1543e-19,\n -2.1817e-17, 2.0048e-19, -5.7634e-19, -4.0994e-17, 2.4406e-19,\n -2.5542e-18, -2.8241e-17, 1.1376e-18, -6.8267e-17, 2.1181e-19,\n -5.2871e-18, -1.6080e-18, 3.6996e-19, 1.2034e-20, -7.5208e-17,\n -2.1891e-17, -5.3622e-17, 2.0341e-19, -3.2514e-19, -4.5371e-17,\n -9.1676e-19, -2.5281e-18, 4.0586e-19, -2.8899e-17, 3.2574e-19,\n -8.3134e-17, -2.1164e-19, -1.8041e-19, -1.8338e-19, -4.7747e-19,\n 3.6223e-20, -6.5274e-18, -3.2131e-18, -1.9237e-17, -1.7263e-17,\n -8.2486e-18, 4.8652e-19, 3.5938e-20, 1.7429e-19, 6.1873e-19,\n -2.7541e-19, -5.4782e-18, -1.2347e-19, -3.5271e-18, -6.9381e-18,\n -8.6108e-17, -6.6178e-17, -5.0309e-17, 1.5914e-19, -2.6215e-19,\n -6.6713e-18, -1.0664e-16, 8.3416e-20, -3.2843e-19, -5.1810e-17,\n 9.2198e-20, -8.3939e-17, -1.1945e-19, -2.1584e-19, 2.7161e-19,\n -4.5816e-19, -1.2261e-18, -4.7751e-17, -3.0043e-18, -9.5819e-19,\n -8.8574e-19, -1.9387e-17, -2.1408e-19, -7.1496e-18, -2.3932e-18,\n -7.0673e-18, -2.3274e-17, -2.0861e-18, -4.2692e-19, -1.4026e-18,\n -5.2125e-18, -9.5977e-17, -9.3807e-17, -1.0887e-19, -4.5215e-18,\n 6.9659e-19, -5.0447e-18, 4.9802e-18, -6.8297e-19, 1.5207e-18,\n -1.4623e-17, -6.2741e-18, -1.2433e-18, -1.5476e-19, 2.8500e-19,\n -8.3007e-20, -1.0080e-17, 2.2971e-19, -8.1172e-17, 2.0611e-18,\n 1.9657e-19, -7.0341e-19, -2.6139e-18, -9.7516e-17, -8.2671e-19,\n -7.3907e-20, -1.2021e-17, -5.1502e-19, -1.0683e-16, -3.2591e-18,\n 2.5790e-18, -4.0110e-17, -1.5465e-18, 1.6557e-19, -9.8080e-18,\n 3.2461e-19, 4.3511e-19, -1.7051e-20, -1.4444e-17, -1.5980e-17,\n -5.0113e-18, -4.4985e-18, 1.5136e-18, -4.2835e-19, -3.8132e-18,\n 4.7771e-19, -1.6283e-19, -1.6214e-17, -9.9540e-18, 5.8495e-19,\n -7.4821e-18, -1.2697e-18, -2.3976e-17, -2.5293e-17, -1.0281e-18,\n 1.5944e-20, -2.2533e-18, -1.2383e-18, -4.9731e-17, 2.5246e-19,\n -6.3641e-19, -3.1572e-20, 3.0948e-19, -3.7417e-17, 1.6490e-18,\n -1.1840e-17, 3.5679e-19, 3.8366e-18, -9.5773e-19, 2.5023e-21,\n 1.4538e-19, -3.7049e-18, -5.3380e-19, -2.2712e-18, -4.8379e-17,\n 3.7935e-20, 5.7418e-20, -6.2268e-20, -1.4844e-19, -6.1219e-17,\n -5.1865e-19], device='cuda:0')", + "exp_avg_sq": "tensor([2.6089e-10, 3.8457e-08, 1.9077e-10, 1.1209e-08, 4.2489e-10, 2.8434e-12,\n 1.2824e-09, 8.1186e-10, 1.5523e-08, 4.9275e-08, 1.8252e-10, 8.1219e-11,\n 7.5358e-12, 7.6869e-11, 3.7393e-09, 1.2000e-08, 7.8724e-09, 7.2899e-13,\n 7.2494e-13, 1.9529e-10, 5.2185e-09, 2.1441e-09, 1.7616e-09, 3.6279e-11,\n 8.5019e-09, 1.4434e-11, 9.3211e-11, 1.2514e-07, 1.2333e-08, 1.0269e-10,\n 1.0274e-11, 4.1685e-08, 5.2143e-12, 8.4667e-12, 4.4930e-11, 8.9339e-13,\n 4.3954e-10, 8.7331e-09, 1.3839e-12, 1.7802e-09, 1.0589e-09, 1.4542e-10,\n 6.8144e-11, 8.8626e-11, 2.1055e-12, 1.8447e-10, 7.8591e-11, 5.1401e-08,\n 2.3418e-11, 1.2359e-09, 1.8467e-11, 6.7583e-13, 1.1229e-07, 9.8841e-09,\n 1.4156e-09, 5.3545e-11, 2.0417e-12, 1.3845e-09, 4.3704e-12, 6.5162e-11,\n 3.1533e-11, 3.0171e-12, 2.3258e-08, 6.6959e-09, 2.0059e-08, 1.2370e-08,\n 2.8288e-10, 6.4143e-11, 5.3035e-12, 8.6847e-11, 7.1838e-09, 2.4178e-12,\n 3.0620e-11, 1.7106e-09, 4.2073e-10, 4.4090e-09, 9.5947e-09, 6.4556e-13,\n 6.8692e-12, 1.4630e-10, 4.4717e-09, 5.1735e-08, 2.0972e-11, 2.0949e-09,\n 7.1245e-12, 1.3322e-09, 3.4518e-10, 1.9591e-10, 6.0291e-11, 1.9334e-12,\n 2.9665e-10, 8.7125e-13, 3.0088e-08, 5.1570e-15, 3.0210e-10, 4.7799e-09,\n 1.0207e-10, 3.1687e-12, 4.6004e-09, 6.8195e-13, 1.7105e-08, 2.3011e-12,\n 4.2091e-12, 2.3208e-09, 7.5848e-11, 3.5642e-12, 1.4477e-10, 3.9194e-13,\n 1.9544e-08, 1.1120e-11, 3.4386e-11, 3.7753e-11, 2.1294e-10, 1.1621e-12,\n 2.2465e-08, 4.8522e-10, 4.9347e-09, 1.1072e-10, 2.3424e-11, 1.4257e-10,\n 1.7086e-09, 1.5325e-13, 8.4437e-12, 8.5312e-10, 6.7386e-14, 3.3846e-08,\n 1.6188e-09, 3.4839e-11, 2.2125e-11, 2.9235e-08, 7.3500e-14, 3.0945e-11,\n 2.8851e-09, 5.5480e-10, 6.9354e-09, 5.6231e-10, 4.5807e-11, 3.8786e-11,\n 3.6800e-11, 5.3112e-12, 2.0152e-12, 9.4166e-10, 3.0838e-12, 1.7680e-11,\n 2.0661e-08, 3.1434e-08, 5.8700e-08, 1.3944e-09, 9.7866e-10, 1.9226e-11,\n 5.3529e-09, 1.0049e-07, 1.2172e-11, 1.1550e-10, 5.8400e-09, 3.2037e-12,\n 3.5906e-08, 7.2126e-11, 5.7119e-10, 3.2638e-12, 1.6043e-12, 3.2930e-10,\n 2.1486e-08, 2.3051e-10, 5.2373e-12, 1.0498e-13, 4.6641e-11, 6.0131e-11,\n 3.4777e-12, 1.0453e-12, 1.0906e-10, 2.2347e-10, 4.6907e-09, 3.2411e-09,\n 3.8054e-11, 1.2319e-10, 5.0075e-08, 6.8665e-08, 3.3986e-12, 1.5736e-10,\n 1.5542e-09, 2.5594e-09, 1.1203e-12, 1.5329e-11, 1.3621e-13, 3.1713e-08,\n 7.0378e-10, 1.0173e-13, 3.0879e-09, 3.9443e-08, 2.7183e-11, 2.2350e-09,\n 5.8475e-12, 1.3347e-08, 5.9897e-12, 2.1960e-12, 1.1931e-10, 2.4257e-10,\n 2.5063e-08, 1.8122e-10, 2.8547e-12, 3.8961e-11, 1.2993e-12, 3.3730e-08,\n 4.5760e-12, 3.7781e-12, 8.7876e-11, 1.4506e-13, 1.8040e-12, 7.5440e-12,\n 1.4191e-10, 6.5012e-13, 8.9043e-10, 1.9343e-09, 8.0210e-10, 3.0817e-10,\n 1.9876e-09, 7.1306e-12, 8.4143e-11, 5.2495e-10, 1.0520e-09, 3.5455e-09,\n 1.9839e-11, 9.5162e-12, 7.0597e-11, 4.8342e-09, 1.0236e-13, 5.4155e-09,\n 1.3589e-09, 1.0864e-12, 1.2558e-11, 1.2834e-10, 1.8949e-10, 6.1118e-09,\n 2.5958e-11, 8.4457e-13, 5.5997e-12, 3.5048e-10, 3.1515e-09, 2.2314e-09,\n 5.1312e-09, 3.7507e-10, 1.1876e-12, 3.3668e-13, 1.4233e-12, 1.5974e-12,\n 1.9195e-10, 6.5775e-12, 2.2772e-10, 2.4353e-08, 1.2256e-10, 1.7245e-11,\n 1.4486e-12, 5.4300e-10, 5.2328e-08, 5.8451e-13], device='cuda:0')" + }, + "51": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-1.2443e-17, -3.3378e-17, -2.6110e-18, -4.5232e-17, -2.1976e-17,\n -4.6189e-19, -2.3512e-17, -9.9435e-18, -6.0706e-17, -4.4242e-17,\n -5.9545e-18, 1.6094e-18, -3.0297e-17, -1.6402e-17, -1.3017e-17,\n -5.6545e-17, -5.4074e-17, -2.0830e-20, -3.3520e-19, -7.5186e-18,\n -3.3749e-17, -2.7490e-17, -1.1941e-17, -1.7202e-17, -3.5585e-17,\n -5.3903e-17, -1.7462e-19, -7.3443e-17, -5.4550e-17, -9.8191e-18,\n 3.4933e-19, -5.0359e-17, -1.9987e-17, -2.3626e-19, 5.2251e-19,\n 4.7358e-19, -8.4565e-18, -2.3356e-17, -2.4574e-18, -4.0961e-18,\n -9.3219e-18, -1.4601e-18, 6.4618e-18, 3.5118e-19, 1.0205e-18,\n 7.5985e-18, -2.7921e-18, -6.9394e-17, -4.4815e-18, -3.9171e-17,\n -2.5825e-18, -3.4729e-19, -3.5498e-17, -1.5033e-18, -1.9457e-17,\n 6.6847e-18, 2.7804e-19, -3.1181e-17, -2.0678e-17, -1.8702e-19,\n -4.4933e-19, -4.3731e-19, -6.0241e-17, -4.3720e-17, -3.0187e-17,\n -7.4354e-18, -4.0138e-17, 2.6315e-18, 2.2878e-18, 3.1402e-19,\n -4.2240e-17, -7.9905e-19, -5.0038e-18, -6.3321e-18, 3.8307e-18,\n -9.0827e-18, -2.8409e-17, -1.0086e-18, -6.2975e-19, -4.0219e-17,\n -1.2249e-17, -9.5263e-18, -8.3841e-19, -5.9271e-18, -3.0466e-17,\n -7.1254e-18, -3.4394e-18, 1.2313e-18, 5.8174e-18, -1.9102e-17,\n 1.8146e-18, 1.9829e-18, -5.3204e-17, 4.8904e-19, 3.8227e-18,\n -3.4507e-17, 5.2411e-19, -2.4475e-17, -1.5701e-17, -4.9275e-18,\n -1.4943e-17, -4.6854e-18, -1.5537e-17, -3.4137e-17, 5.2347e-19,\n -1.4081e-17, -3.1946e-17, -9.3141e-19, -5.7117e-17, -2.4786e-18,\n 3.8752e-18, 8.7782e-18, -2.1201e-18, -1.2859e-19, -4.2964e-17,\n -2.8149e-17, -4.5602e-17, 1.1947e-18, -3.0590e-21, -5.0142e-17,\n -8.5049e-18, 2.0251e-18, -3.6757e-18, -2.8643e-17, -2.7612e-19,\n -4.6115e-17, 2.5727e-18, 1.3117e-18, 3.2273e-19, -4.3232e-18,\n 5.2923e-19, 4.8597e-18, -7.7833e-18, -3.0876e-17, -2.6885e-17,\n 6.2692e-18, -2.2899e-18, -2.2440e-19, 1.7332e-18, 6.6332e-20,\n 1.6965e-18, 3.5734e-18, 4.8466e-19, 2.4980e-18, -1.5654e-17,\n -3.8883e-17, -3.5831e-17, -4.8980e-17, -2.2757e-18, 1.1822e-18,\n -2.1006e-17, -5.8347e-17, 1.6275e-18, 1.6595e-18, -3.6733e-17,\n -6.7370e-20, -5.2475e-17, -2.6768e-19, 2.5278e-18, -3.2323e-18,\n 3.7906e-19, -1.8685e-18, -2.5730e-17, 2.4414e-18, -8.1961e-18,\n 7.2463e-19, -2.6405e-17, -2.1890e-18, -1.6384e-17, 1.7910e-18,\n 4.9989e-18, -2.7259e-17, -3.8311e-18, -3.8010e-18, 3.1423e-19,\n -9.2000e-18, -4.5830e-17, -4.4760e-17, 1.7781e-19, -5.7850e-18,\n -9.8948e-18, -1.3929e-17, -3.7870e-18, 4.3749e-19, -1.2806e-18,\n -2.2219e-17, -1.6484e-17, 7.8698e-19, -3.4547e-18, 6.8041e-19,\n -2.1984e-19, -1.1680e-17, 2.3005e-19, -5.8099e-17, -1.5783e-18,\n -1.6772e-18, -7.0803e-19, 1.6949e-18, -5.8849e-17, 3.3256e-19,\n 7.0591e-19, -2.6588e-17, -1.5588e-19, -6.0418e-17, 1.0246e-18,\n -1.9650e-18, -5.3141e-17, 1.2107e-18, -1.1882e-18, -1.6428e-17,\n -2.9273e-18, -2.8998e-18, -1.0115e-18, -1.9923e-17, -2.0660e-17,\n 3.3354e-18, -5.8624e-18, -1.2601e-18, 4.0108e-18, -6.8501e-18,\n -4.6097e-18, -3.1634e-18, -2.5963e-17, -1.8373e-17, -2.4405e-18,\n -2.1775e-17, 8.7955e-19, -4.1661e-17, -3.5848e-17, 2.2695e-19,\n -1.2111e-19, 1.6291e-18, 1.4811e-18, -3.7112e-17, -9.2730e-19,\n -1.3347e-19, 2.1028e-20, -7.0246e-18, -3.1425e-17, 1.8415e-18,\n -1.9726e-17, -3.3283e-18, -3.0403e-18, 8.0247e-19, -2.3424e-20,\n 4.8020e-19, 2.9325e-18, -5.8806e-18, 1.7773e-19, -2.5958e-17,\n 5.7878e-19, -9.5764e-20, -1.0139e-18, -7.3770e-18, -3.4206e-17,\n 3.9757e-19], device='cuda:0')", + "exp_avg_sq": "tensor([1.7567e-09, 2.8665e-08, 6.3710e-10, 1.9380e-08, 4.1268e-09, 1.5655e-12,\n 2.1301e-09, 9.7218e-10, 1.6046e-08, 1.5394e-08, 1.0323e-09, 4.0253e-11,\n 2.8117e-09, 7.3395e-10, 1.8326e-09, 1.6711e-08, 2.5342e-08, 3.0190e-11,\n 1.1354e-13, 7.5000e-10, 7.3052e-09, 2.3708e-09, 1.9296e-09, 1.5443e-09,\n 5.0308e-09, 1.6734e-09, 4.4544e-11, 4.9101e-08, 7.0364e-09, 1.0919e-09,\n 2.1886e-09, 2.0820e-08, 1.0071e-09, 4.6463e-12, 8.0374e-10, 4.3840e-10,\n 6.5073e-10, 9.5227e-09, 9.2540e-11, 2.5378e-09, 1.3616e-09, 6.4897e-11,\n 3.3307e-11, 3.3788e-11, 1.4269e-10, 7.7553e-11, 4.0526e-11, 1.9073e-08,\n 1.0388e-10, 4.4137e-09, 1.0098e-11, 1.3453e-11, 4.3093e-08, 1.2712e-08,\n 2.2640e-09, 4.3216e-11, 4.5392e-13, 4.2642e-09, 1.4099e-09, 2.8523e-11,\n 1.2001e-09, 1.4316e-10, 2.5630e-08, 1.6742e-08, 1.8731e-08, 7.7265e-09,\n 2.0771e-09, 3.3221e-11, 1.5581e-12, 1.8755e-09, 1.9311e-08, 2.2037e-13,\n 4.9988e-10, 1.0533e-09, 1.6822e-10, 2.5162e-09, 5.1423e-09, 3.3917e-12,\n 2.8105e-10, 5.9230e-09, 2.3799e-09, 2.0379e-08, 6.9627e-10, 1.3141e-09,\n 4.5954e-10, 7.1279e-10, 4.7995e-10, 8.0386e-11, 1.2957e-09, 4.0932e-10,\n 1.6408e-10, 1.2915e-11, 1.3268e-08, 1.6901e-13, 1.5309e-10, 5.0909e-09,\n 5.3907e-11, 6.2175e-10, 4.2444e-09, 3.4114e-10, 1.3815e-08, 1.9113e-10,\n 5.0504e-11, 3.2351e-09, 1.4227e-09, 8.2380e-11, 2.6398e-09, 2.6315e-13,\n 1.1900e-08, 5.3164e-12, 9.3941e-12, 1.7552e-11, 7.7785e-10, 2.5783e-10,\n 1.4396e-08, 2.8764e-09, 1.0230e-08, 5.7376e-09, 1.0624e-11, 2.3542e-09,\n 2.6048e-09, 2.1288e-12, 4.5193e-12, 6.7585e-09, 9.7945e-12, 1.9850e-08,\n 5.9265e-09, 2.5706e-10, 5.6474e-10, 1.9874e-08, 3.7810e-11, 1.2977e-11,\n 1.9677e-09, 2.8426e-09, 2.8999e-09, 2.4124e-10, 2.5475e-11, 2.0128e-11,\n 1.6314e-11, 1.5922e-12, 1.0302e-12, 3.8406e-10, 1.1979e-12, 4.7057e-12,\n 1.1972e-08, 1.0371e-08, 2.9433e-08, 9.1123e-09, 3.0405e-09, 9.5666e-12,\n 4.6553e-09, 5.7236e-08, 3.8285e-11, 3.8309e-11, 7.2932e-09, 2.0856e-11,\n 3.1024e-08, 3.8153e-11, 8.9968e-09, 1.7975e-10, 4.9254e-10, 1.3615e-10,\n 8.7063e-09, 1.1376e-10, 3.3293e-10, 5.6788e-12, 1.5105e-09, 2.6267e-11,\n 6.7925e-10, 5.5936e-11, 4.7661e-11, 2.7221e-09, 2.1032e-09, 3.8877e-09,\n 1.6682e-11, 1.1396e-09, 1.6748e-08, 2.2886e-08, 2.6775e-13, 4.3053e-09,\n 4.0765e-09, 2.1463e-09, 6.6067e-11, 7.5690e-12, 6.6388e-12, 1.2832e-08,\n 5.1692e-10, 1.9793e-12, 3.9438e-09, 2.7148e-08, 1.3914e-11, 3.1257e-09,\n 1.4894e-10, 1.6365e-08, 6.5362e-10, 5.9252e-11, 5.6977e-11, 9.4908e-11,\n 1.7686e-08, 7.7564e-11, 5.0295e-13, 2.9299e-09, 6.4483e-11, 2.1627e-08,\n 5.4196e-12, 2.4950e-10, 3.4191e-09, 7.1211e-12, 7.1753e-11, 1.9177e-09,\n 6.9147e-11, 1.9335e-10, 1.3830e-09, 9.5481e-10, 2.0085e-09, 1.6415e-10,\n 1.6868e-09, 6.2489e-12, 4.3754e-11, 2.5895e-10, 2.8129e-09, 5.4638e-09,\n 1.3950e-09, 1.8871e-09, 3.0381e-11, 4.6380e-09, 8.0035e-12, 3.7242e-09,\n 4.6596e-09, 1.6011e-10, 1.0273e-09, 6.0769e-11, 9.9195e-11, 7.7209e-09,\n 2.0672e-10, 6.9456e-11, 1.2267e-09, 1.8579e-10, 7.2659e-09, 3.5471e-09,\n 3.7287e-09, 1.9313e-10, 5.4743e-11, 1.0318e-11, 2.1465e-11, 1.9648e-14,\n 1.0370e-10, 1.5421e-11, 8.7216e-11, 1.0731e-08, 3.8020e-10, 1.6373e-12,\n 3.1608e-10, 1.0859e-09, 2.9677e-08, 1.0004e-10], device='cuda:0')" + }, + "52": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[ 5.9806e-19, -7.0037e-19, -3.9747e-19, ..., 9.9257e-21,\n 9.4937e-19, -3.5135e-19],\n [ 2.3992e-19, -1.1057e-18, 6.0436e-20, ..., -6.0690e-21,\n -7.1238e-19, 7.1075e-20],\n [-3.2094e-19, 5.1797e-19, 4.5062e-19, ..., 7.9445e-20,\n -1.8968e-18, 3.7562e-19],\n ...,\n [ 4.1105e-18, -7.1305e-17, 1.2167e-17, ..., 7.9649e-18,\n -1.0443e-16, 3.8541e-18],\n [ 1.0055e-18, -8.2392e-18, 2.1657e-18, ..., 1.8089e-18,\n -1.5710e-17, 9.1100e-19],\n [-2.6866e-18, 3.8741e-17, -8.2527e-18, ..., -6.1411e-18,\n 6.2776e-17, -1.6192e-18]], device='cuda:0')", + "exp_avg_sq": "tensor([[5.5161e-10, 9.5222e-10, 3.1164e-10, ..., 1.2530e-10, 1.1663e-09,\n 3.7049e-09],\n [2.4803e-11, 1.8864e-11, 2.1350e-12, ..., 2.2086e-11, 2.5653e-11,\n 1.1385e-10],\n [1.0342e-10, 7.0354e-11, 5.4549e-13, ..., 5.5548e-11, 1.4719e-10,\n 3.1181e-10],\n ...,\n [1.3403e-09, 2.1568e-10, 1.8784e-11, ..., 3.5293e-11, 1.3228e-09,\n 1.5294e-09],\n [8.6248e-08, 1.0717e-08, 3.0018e-10, ..., 6.3057e-09, 3.8084e-08,\n 1.0757e-07],\n [1.9128e-08, 6.9529e-09, 5.6513e-11, ..., 1.7145e-09, 2.0379e-08,\n 3.3040e-08]], device='cuda:0')" + }, + "53": { + "step": "tensor(1252.)", + "exp_avg": "tensor([ 2.2505e-18, -6.8211e-19, -2.7778e-18, 2.7497e-18, 8.7213e-18,\n 9.3086e-20, 6.5669e-18, -1.3943e-18, 5.5198e-18, 3.9901e-18,\n 1.2580e-19, 3.0178e-18, -8.7836e-18, 5.0338e-19, -2.9999e-18,\n -6.7223e-18, 1.3609e-18, -4.2494e-18, -5.2085e-18, 5.9407e-20,\n -4.6224e-18, 1.1854e-19, 6.3222e-18, 3.1683e-20, -8.5099e-19,\n -3.4360e-18, 7.9427e-18, -8.8789e-18, -4.6254e-18, -2.6147e-19,\n 5.3735e-18, 5.4157e-18, -3.2713e-18, 3.0188e-18, 1.7825e-19,\n 2.0301e-18, -2.4345e-18, -1.3722e-18, 5.9184e-19, 3.4073e-18,\n -1.4292e-18, 1.2087e-18, -1.4356e-18, 2.1358e-18, -2.4285e-18,\n 2.1093e-18, 5.9655e-18, 1.5356e-18, -2.5268e-18, -4.8649e-19,\n -2.2345e-18, -5.6438e-19, 1.1632e-19, -1.3540e-18, 7.6948e-19,\n -9.5332e-19, 1.6197e-18, 3.1150e-18, -9.8883e-19, 9.6683e-19,\n 2.8711e-18, 7.1653e-19, -1.1762e-19, 5.9965e-19, -1.0812e-18,\n 4.5754e-19, 7.1911e-19, -1.6079e-18, 1.9790e-18, 1.7251e-18,\n 6.4415e-19, -1.5158e-18, -3.5113e-18, 2.3737e-19, -2.7210e-19,\n 6.0979e-19, 5.0399e-19, 1.6607e-18, 6.0826e-19, 6.2906e-19,\n 2.9182e-19, 8.8107e-19, -5.1293e-19, 2.5860e-18, 8.0234e-19,\n -2.3176e-18, -9.2500e-19, 1.8031e-18, 6.7121e-19, -4.3407e-20,\n 2.7737e-18, -5.5807e-19, -1.1683e-18, -1.2297e-18, -9.5943e-19,\n -1.8891e-18, -4.4932e-18, -4.0386e-19, 2.1083e-18, -1.8249e-18,\n -1.2404e-18, 2.2240e-18, 3.4127e-18, -1.2408e-18, 4.9175e-18,\n -1.6289e-18, 8.8723e-19, 1.5972e-18, -4.8959e-18, 3.5676e-18,\n 1.0007e-18, 2.8197e-18, 7.5530e-19, -7.1079e-19, 4.9270e-18,\n 5.2654e-18, 2.3261e-18, 5.5602e-18, -3.0049e-18, -4.6720e-19,\n 2.0624e-18, -8.6984e-19, -1.2974e-18, -1.6866e-18, -3.2761e-18,\n 1.3196e-18, -1.8474e-18, 2.9508e-18, 4.4954e-19, -7.1552e-19,\n -1.0901e-18, 2.0120e-18, 8.6780e-19, -6.7511e-19, 1.5174e-18,\n 5.4827e-19, 9.2871e-19, -2.0172e-18, -1.1738e-18, -1.9483e-18,\n 6.9757e-19, 4.4908e-19, 5.5050e-19, 4.7532e-19, 3.0379e-18,\n -1.2956e-18, 1.1631e-18, -2.0254e-18, 1.4521e-18, -1.2573e-18,\n 1.0821e-18, 2.2532e-18, 7.3622e-19, -1.7291e-18, 1.8125e-18,\n 9.3518e-19, 3.7322e-18, 2.7159e-21, -2.1443e-18, 3.0122e-19,\n 1.7317e-19, -2.3209e-18, -4.4478e-18, 4.5196e-19, -6.9238e-19,\n -7.4867e-19, 3.3806e-19, 8.7608e-18, 1.0994e-18, -2.9033e-18,\n -3.2396e-18, -2.6942e-18, -6.3771e-19, -2.9912e-18, -5.6468e-18,\n -5.1022e-18, -2.9977e-18, -1.8143e-18, -6.3863e-18, 4.9597e-18,\n 3.8888e-18, -4.1997e-18, -6.7577e-19, 3.9356e-18, 2.5015e-18,\n -4.1501e-18, 7.5515e-19, 4.0182e-18, -2.5659e-18, 1.6637e-18,\n 1.6978e-18, 2.8103e-18, 1.6614e-18, -3.3323e-19, -1.1893e-18,\n 2.2347e-18, -1.2417e-18, -1.8309e-18, 8.2604e-19, -4.7148e-19,\n 1.4514e-18, 1.2380e-18, 2.4378e-18, -2.0116e-18, -6.4088e-19,\n -1.1317e-18, 2.2520e-18, 9.8215e-19, 2.2840e-18, -9.3644e-19,\n -2.1786e-19, -2.9179e-19, -3.3835e-18, 1.6424e-18, 2.3183e-18,\n 2.5301e-18, 3.8265e-19, -2.2481e-19, 3.9398e-19, 6.8235e-19,\n -1.2730e-18, -1.2118e-18, -4.0364e-18, 2.5047e-18, 2.6994e-18,\n 2.7584e-19, -4.2011e-20, -2.9361e-18, 1.3413e-18, 1.1005e-19,\n -3.7446e-18, -1.1798e-18, -3.4731e-18, -2.5081e-18, 1.9306e-18,\n -3.9083e-18, 8.4671e-18, -5.2635e-19, -8.9250e-19, 6.0165e-18,\n -2.5048e-18, 4.4573e-19, -3.8993e-18, 3.6296e-19, -2.1236e-18,\n -7.9836e-18, -3.0087e-18, 5.2929e-18, -1.0525e-18, 5.2518e-18,\n 1.9612e-18, 4.9410e-19, 2.4455e-19, -2.5824e-18, 2.5903e-19,\n 3.8386e-20, -2.3437e-25, 1.0540e-24, -2.5206e-25, 7.5085e-25,\n 9.9498e-25, -6.3448e-26, 1.3226e-24, 3.8072e-25, 6.2652e-25,\n 7.3468e-25, -5.3359e-25, 7.5657e-25, -2.3241e-24, -2.8924e-25,\n -3.0964e-25, -8.7477e-25, -1.1602e-25, -6.4787e-25, 3.3944e-25,\n -1.2165e-25, -2.3865e-25, -6.1632e-25, 1.0772e-24, -8.4588e-25,\n -4.5322e-25, 9.0586e-27, 7.9920e-25, -6.3377e-25, -8.7420e-25,\n 4.3747e-25, -1.1625e-25, 2.0700e-24, -2.7868e-25, -5.0653e-26,\n 2.5046e-25, 1.7574e-25, -1.2810e-24, -7.9628e-26, -5.5252e-25,\n 6.5728e-25, -1.5112e-24, 7.2440e-25, -6.6602e-26, 1.2159e-24,\n -1.0363e-24, 8.9836e-25, 1.5832e-24, 8.7382e-25, 7.5960e-25,\n 1.2856e-24, -1.3814e-24, -3.2250e-25, 7.8622e-25, 6.3573e-25,\n 3.8326e-25, 1.8678e-25, 9.6297e-25, 1.2773e-24, -5.6197e-25,\n -8.6652e-25, 1.6554e-24, -1.5558e-24, -3.8824e-25, -1.7651e-25,\n -6.5489e-25, 2.5710e-25, -4.2353e-25, -4.5702e-25, 2.8080e-25,\n 1.4509e-25, -4.6169e-25, -5.5279e-25, -3.9488e-25, 9.5551e-25,\n 3.9921e-25, -6.6898e-26, 4.2500e-25, -9.7699e-26, 3.8362e-25,\n -2.9822e-25, -6.0612e-25, 9.8724e-26, -5.1442e-25, 1.2106e-24,\n 1.2360e-25, -9.7411e-26, -3.3223e-25, 4.3943e-25, -7.2429e-25,\n 3.6747e-25, -2.0010e-25, 2.5526e-26, 3.4202e-25, 5.7298e-25,\n -1.0458e-24, 2.5758e-25, -1.6036e-24, -1.3132e-25, -4.9334e-25,\n -1.0455e-24, -1.8315e-24, 6.0049e-25, 1.0373e-24, -5.8408e-25,\n 9.3175e-25, 7.0031e-26, -3.6449e-26, 5.5135e-25, -5.7195e-25,\n 1.7498e-24, -3.2164e-25, -3.2937e-25, 3.1129e-25, -7.6103e-26,\n 1.0009e-25, 1.0148e-24, -6.1792e-25, 1.4461e-24, -7.3255e-25,\n 2.0460e-25, -5.9593e-25, -4.9279e-25, -4.8601e-25, 4.6113e-25,\n -2.0937e-25, 3.7193e-25, -7.2168e-25, 6.5140e-25, 2.8704e-25,\n 2.0418e-24, -9.5845e-25, -1.1758e-24, 1.1309e-24, -1.1369e-24,\n -3.3955e-25, -9.7839e-25, 2.2527e-25, 3.4282e-26, 1.3729e-24,\n 1.0656e-24, 1.3777e-24, -1.6256e-24, -2.4464e-24, -2.2901e-24,\n -9.4101e-25, 6.0233e-25, 5.5720e-25, 2.1947e-24, 8.1770e-26,\n 9.3228e-25, -3.0283e-25, -5.9494e-25, -9.2388e-26, 1.7092e-24,\n 6.3045e-25, -2.6503e-25, -8.9099e-25, 1.0101e-25, 1.0784e-24,\n 3.8646e-25, 7.7302e-25, -4.8427e-26, -3.3435e-25, -7.0921e-27,\n 3.6107e-25, 2.0863e-25, 2.2454e-25, 4.8569e-25, 3.5095e-25,\n -7.9588e-27, -9.6670e-25, 1.2335e-25, -4.0150e-25, -5.6414e-25,\n 1.7607e-25, -5.0960e-25, 1.1298e-25, 4.3684e-25, -6.4323e-25,\n 5.7012e-26, -2.5084e-25, -6.1652e-25, 4.8946e-26, 8.1896e-25,\n -3.3010e-25, -4.3621e-25, -5.1901e-25, -5.8795e-25, 2.5170e-25,\n 1.1091e-24, -1.8792e-25, 4.6972e-26, -4.5400e-25, -7.7524e-25,\n -4.8085e-25, 1.4234e-24, 5.4167e-25, -6.2715e-25, 5.3988e-27,\n 5.8605e-25, 1.0848e-24, -3.8264e-25, -3.6424e-25, 5.3968e-25,\n 4.4375e-25, 1.7094e-25, -5.4883e-26, 1.6882e-25, -3.0195e-25,\n -1.9085e-24, 8.7634e-25, -9.6086e-25, -1.3285e-24, 7.2236e-25,\n 1.6756e-26, 7.0150e-25, -9.7971e-25, 1.1740e-25, 3.5850e-25,\n -5.0923e-25, 3.6027e-25, -6.6392e-25, 7.4705e-26, 1.5178e-26,\n -2.0125e-27, 1.0876e-24, -7.3213e-25, 8.7879e-25, 4.8837e-25,\n 7.5529e-25, -1.3866e-24, -1.3651e-24, 5.6178e-25, 4.0978e-25,\n -1.1662e-24, 5.5280e-25, -7.4023e-25, 1.7683e-24, 5.3211e-25,\n -2.5243e-25, -7.6594e-25, 2.2514e-24, 1.9603e-25, -1.2198e-24,\n -6.2896e-25, 2.2680e-24, 1.0745e-24, -1.6827e-24, -8.6020e-25,\n 3.3311e-25, -3.1333e-24, -1.9292e-24, 6.7031e-25, -5.9474e-25,\n -5.7107e-27, -1.7721e-25, 5.4095e-18, 6.5317e-17, -6.8547e-17,\n -2.9185e-17, 1.2609e-16, -1.1119e-16, -1.1520e-16, 2.9812e-17,\n -2.0995e-17, -1.3823e-17, -7.2491e-17, 7.2026e-17, -4.3277e-17,\n 7.6475e-17, 8.7381e-17, 4.2183e-17, -1.0522e-16, 7.3177e-17,\n -6.4292e-17, 9.6868e-17, -1.0683e-16, 9.1063e-17, -1.1509e-16,\n -1.0388e-16, -6.0704e-17, -1.9276e-17, -1.5656e-16, -9.1062e-17,\n 6.5944e-17, -9.7136e-17, 9.5972e-17, 6.4289e-17, -7.6704e-17,\n 8.3662e-17, -6.7260e-17, -3.7841e-17, 1.4192e-16, 4.2325e-17,\n -1.4150e-17, -1.0112e-16, 1.4518e-17, 1.5801e-16, 1.2234e-16,\n 6.9944e-17, -2.7549e-17, -9.0413e-17, 1.3293e-16, -1.4677e-16,\n -1.0706e-16, -7.9190e-17, 5.9676e-17, -3.8734e-17, 2.6727e-17,\n -9.5679e-17, -3.8548e-17, 8.3681e-17, -8.0507e-17, 8.7855e-17,\n -6.9184e-17, -3.7454e-17, 1.4281e-16, -7.6683e-17, 1.2454e-16,\n 4.4304e-17, 1.3617e-16, 2.5280e-17, -8.2204e-17, 8.5641e-17,\n -1.1688e-16, 1.6402e-16, 6.7500e-18, -1.1739e-16, 8.6436e-17,\n -4.4853e-17, 8.4608e-17, -6.3421e-17, 5.0107e-17, -3.0924e-17,\n 1.1488e-17, -5.2973e-17, 3.4089e-17, -7.6261e-17, 6.6235e-17,\n 8.6613e-17, 7.5757e-17, 2.5517e-17, -2.8805e-17, -3.5784e-17,\n 7.9015e-17, -5.4724e-17, -9.9688e-17, -7.9864e-17, 6.0292e-17,\n 1.1382e-16, -3.3388e-17, 4.8807e-17, -5.9528e-17, 1.4785e-16,\n -7.5916e-18, 7.2342e-17, 2.0765e-17, 7.0356e-17, 4.2273e-17,\n -1.5470e-16, 8.8031e-17, -1.3463e-17, 7.6329e-17, -9.2821e-18,\n -7.9835e-17, -6.4274e-17, 1.3050e-16, 5.3795e-17, 6.3990e-17,\n 9.2628e-17, 7.7467e-17, 6.7179e-17, 1.2180e-16, -1.2875e-16,\n 1.1457e-16, -5.9667e-17, 1.4912e-16, 6.4871e-17, 1.5327e-16,\n 3.4272e-17, 1.3444e-16, -1.5049e-16, -6.9932e-17, 7.8600e-18,\n -1.4371e-16, 2.2237e-17, -1.0033e-16, 9.1513e-17, 2.9005e-17,\n 7.3682e-17, -4.3495e-18, -1.0460e-16, 3.5838e-17, -1.1563e-16,\n 1.2098e-16, -7.7314e-17, -1.5801e-16, 3.2399e-17, -3.5520e-17,\n -6.6660e-17, -5.6201e-17, -7.9787e-17, 9.2763e-17, 5.8542e-17,\n -2.3611e-17, 3.8581e-17, 3.7920e-17, 9.0332e-17, -1.0951e-16,\n 9.3559e-18, -9.1297e-17, -7.3266e-17, -1.0982e-16, -7.9323e-17,\n -7.8665e-17, -1.0770e-16, 1.2427e-16, 7.4388e-17, -3.8013e-17,\n 5.0983e-17, -1.0456e-16, 2.8826e-17, -3.4026e-17, -5.1698e-17,\n 6.7565e-17, 1.5212e-16, -1.0200e-16, -1.1418e-17, 5.6879e-17,\n -8.3309e-17, -5.4922e-17, 2.1494e-17, -1.7098e-16, -1.0832e-16,\n 5.7446e-17, -1.1806e-16, -3.7620e-17, -3.6699e-17, 2.6113e-17,\n -1.0288e-16, 2.5459e-17, -4.0819e-17, 9.3112e-17, 3.8454e-17,\n -6.3563e-17, -8.0712e-17, 4.2519e-18, -8.9322e-17, -2.2464e-17,\n -9.9075e-17, 6.1567e-17, -1.0423e-16, 3.4172e-17, -1.2054e-16,\n 1.6622e-17, 2.1879e-17, 2.7696e-17, 1.2410e-16, 1.2889e-16,\n 9.0293e-17, -2.9287e-17, -8.2712e-17, -4.2359e-17, 1.2701e-16,\n -5.0839e-17, -1.0850e-16, 6.4063e-17, 1.0797e-16, -1.3059e-16,\n 4.7609e-17, 5.0808e-17, -7.5151e-17, 4.0434e-18, -2.0042e-17,\n -3.7151e-17, 7.3184e-18, 2.0748e-17, -8.2564e-17, 5.8079e-17,\n 1.1425e-16, -1.4882e-16, -4.4422e-18, -1.0623e-16, 8.5332e-17,\n 1.1344e-16, 2.8827e-17, -5.2204e-17, -1.1956e-16, -1.1985e-16,\n -8.7461e-17, 5.4730e-17, -4.4940e-18, 7.5386e-17, -1.2016e-16,\n 7.1575e-17, -8.8395e-17, -4.7577e-17, -1.1690e-16, -5.1141e-17,\n -1.1756e-16, -1.2670e-16, -1.5483e-17, 7.8899e-17, -1.1571e-16,\n -7.6584e-17, 5.5728e-17, -5.7865e-17, -1.0181e-16, 7.9518e-17,\n -1.0411e-16, -1.8423e-17, 6.9744e-17], device='cuda:0')", + "exp_avg_sq": "tensor([1.3345e-09, 1.0269e-10, 7.9542e-10, 5.4982e-10, 6.9546e-10, 1.1363e-09,\n 1.2655e-10, 1.4666e-09, 1.8983e-09, 1.3398e-10, 7.4478e-10, 2.6191e-10,\n 1.8758e-09, 4.2362e-11, 2.1602e-09, 3.0066e-10, 4.2857e-10, 3.7777e-10,\n 4.3129e-10, 2.1613e-10, 2.5073e-10, 5.0914e-10, 4.5536e-11, 6.0229e-10,\n 8.0608e-10, 5.4085e-11, 8.2640e-10, 2.7667e-09, 1.6113e-10, 2.7573e-11,\n 7.8660e-10, 9.0492e-10, 3.4304e-09, 1.3203e-11, 2.9211e-10, 3.8371e-10,\n 2.4776e-09, 9.9024e-12, 5.1108e-10, 1.6858e-10, 2.4428e-10, 3.6865e-11,\n 3.4454e-09, 9.6904e-10, 2.1443e-09, 9.8349e-11, 5.3314e-09, 2.1023e-09,\n 6.0722e-10, 6.0287e-11, 1.6717e-10, 3.1269e-09, 8.6238e-11, 5.8767e-10,\n 2.4014e-11, 1.0885e-10, 2.0485e-09, 4.7942e-10, 2.6770e-10, 1.9305e-09,\n 1.5220e-09, 3.8966e-09, 3.3234e-09, 8.5101e-10, 1.9754e-09, 1.4393e-10,\n 3.6977e-10, 5.4249e-10, 4.3775e-11, 2.7348e-09, 2.9141e-09, 2.2036e-10,\n 2.3881e-09, 9.4575e-11, 2.2397e-10, 3.0923e-09, 3.6148e-10, 1.0728e-09,\n 4.6106e-10, 1.4308e-09, 6.5912e-10, 1.0494e-10, 6.0189e-10, 2.6825e-09,\n 2.4378e-09, 6.8533e-10, 3.4266e-10, 1.4690e-09, 6.3678e-10, 1.8764e-10,\n 1.1335e-09, 7.1045e-10, 7.6024e-10, 3.5351e-11, 2.0934e-09, 4.9269e-10,\n 2.4127e-10, 7.7012e-11, 7.6878e-10, 2.5803e-10, 8.7434e-11, 9.3497e-11,\n 3.0000e-10, 4.3322e-10, 1.9125e-10, 2.2959e-10, 1.0088e-10, 7.2704e-10,\n 1.7160e-10, 3.2941e-10, 2.0886e-11, 9.4405e-10, 1.3600e-09, 1.5900e-10,\n 2.4929e-10, 1.1027e-10, 4.5967e-10, 4.7937e-11, 5.1488e-11, 1.7059e-10,\n 5.3424e-11, 6.9332e-11, 2.1398e-10, 3.9021e-10, 2.8249e-10, 1.2886e-11,\n 4.5611e-10, 6.1321e-11, 2.2018e-10, 4.0903e-10, 2.4838e-09, 2.2524e-09,\n 1.5810e-09, 7.1230e-10, 8.3922e-10, 6.0728e-11, 3.6522e-09, 1.1497e-09,\n 6.6237e-10, 4.1238e-10, 9.4227e-10, 1.1285e-10, 1.4290e-10, 1.4242e-10,\n 9.7892e-11, 2.6182e-09, 2.0804e-10, 1.1102e-10, 7.1581e-09, 2.5449e-10,\n 6.8462e-10, 3.4413e-09, 1.6855e-09, 4.7146e-11, 2.1991e-10, 1.1487e-11,\n 3.7160e-09, 1.5048e-10, 1.2760e-09, 2.0505e-09, 6.0731e-12, 7.4610e-12,\n 7.1733e-11, 1.9987e-10, 1.9303e-11, 1.3290e-10, 2.3992e-11, 5.6278e-10,\n 9.7241e-11, 1.1293e-10, 6.6692e-11, 2.5193e-11, 1.4218e-10, 1.9523e-11,\n 1.4766e-11, 6.1120e-11, 6.2413e-12, 2.1544e-10, 4.9626e-11, 1.1157e-10,\n 2.4852e-10, 1.5367e-10, 4.3117e-11, 4.3320e-11, 8.6354e-12, 3.5506e-11,\n 2.4557e-10, 1.3132e-10, 9.3428e-11, 1.1137e-10, 1.7527e-10, 1.8172e-10,\n 2.6055e-10, 2.4679e-09, 5.8752e-09, 4.7712e-09, 2.3622e-09, 8.5397e-11,\n 7.0887e-10, 3.8375e-11, 1.2120e-09, 4.8590e-09, 6.7040e-10, 9.1414e-10,\n 9.1878e-10, 2.5578e-10, 8.8425e-10, 3.4020e-11, 3.8033e-09, 8.9830e-10,\n 2.8456e-09, 1.0256e-10, 1.5282e-09, 3.2670e-11, 6.8332e-12, 6.2946e-09,\n 4.9305e-10, 6.7712e-10, 1.1059e-10, 6.5232e-09, 1.5010e-10, 4.1062e-09,\n 1.9750e-09, 1.2749e-11, 4.6588e-10, 2.6517e-10, 1.4350e-10, 6.2043e-10,\n 1.5295e-10, 8.4202e-12, 1.0125e-11, 3.2398e-10, 2.2706e-10, 1.7015e-10,\n 1.0006e-09, 6.2439e-11, 4.9869e-10, 3.3949e-10, 1.2822e-11, 6.8542e-11,\n 4.9668e-10, 1.0195e-09, 3.2133e-10, 1.4493e-09, 7.4357e-11, 2.3705e-10,\n 2.2852e-09, 6.2918e-10, 5.0849e-11, 1.8763e-10, 4.0314e-10, 1.8478e-10,\n 2.0938e-10, 2.2563e-10, 1.6135e-11, 4.4356e-11, 9.1298e-27, 2.1189e-25,\n 1.2994e-25, 2.4193e-26, 3.5147e-26, 6.5019e-26, 3.9030e-26, 1.1288e-26,\n 2.3049e-26, 1.2580e-26, 8.3920e-26, 9.9600e-27, 7.9086e-26, 1.0562e-26,\n 1.5620e-26, 7.1173e-26, 2.9564e-26, 3.8506e-26, 3.7986e-26, 6.2070e-27,\n 1.8226e-25, 3.9213e-26, 1.7481e-25, 1.4975e-25, 1.3986e-25, 2.1378e-26,\n 1.1170e-25, 4.6875e-26, 1.0296e-25, 2.9107e-26, 1.4216e-26, 2.1189e-25,\n 8.3478e-26, 1.2414e-26, 6.5205e-26, 1.2725e-25, 2.1213e-25, 3.8255e-26,\n 9.2860e-26, 1.9240e-25, 2.9481e-26, 2.6777e-26, 1.4028e-25, 1.0194e-25,\n 7.1663e-26, 9.3160e-26, 1.7519e-25, 3.9775e-26, 7.1363e-26, 3.2035e-26,\n 2.0377e-26, 5.8739e-26, 1.6644e-26, 3.2840e-26, 8.3828e-26, 1.1580e-25,\n 2.4020e-25, 8.2401e-26, 1.1212e-25, 8.6942e-26, 1.3955e-25, 1.5834e-25,\n 4.3839e-26, 6.8082e-26, 3.6142e-25, 2.8067e-26, 2.1107e-25, 1.2923e-25,\n 9.6051e-26, 5.3932e-26, 3.6825e-26, 8.2488e-26, 1.2078e-25, 5.2178e-25,\n 1.9202e-26, 1.3708e-26, 1.4138e-26, 2.4075e-27, 3.8120e-26, 3.0799e-26,\n 1.6104e-26, 4.2518e-26, 1.7008e-25, 6.8802e-26, 2.8168e-26, 3.9592e-26,\n 5.9861e-26, 5.8283e-26, 2.6650e-26, 4.9834e-26, 6.7228e-27, 3.6207e-26,\n 9.2663e-27, 3.4704e-26, 4.8720e-26, 4.1825e-26, 5.4504e-26, 1.1834e-26,\n 3.0104e-26, 5.3537e-26, 2.8461e-26, 9.3222e-27, 1.3516e-26, 2.7402e-26,\n 5.6408e-26, 2.8288e-26, 8.1301e-27, 4.9630e-26, 1.2303e-26, 2.0550e-26,\n 2.7685e-26, 3.9208e-26, 4.0734e-26, 1.2602e-26, 1.0688e-25, 3.9249e-26,\n 1.8651e-26, 2.6674e-26, 1.8738e-26, 5.4270e-26, 2.3912e-26, 3.6812e-26,\n 2.2351e-26, 1.9408e-26, 2.0304e-26, 2.5226e-26, 1.4836e-26, 1.0550e-26,\n 2.2359e-25, 1.2096e-25, 5.5531e-26, 1.3196e-25, 3.1651e-25, 1.5823e-25,\n 3.2728e-26, 2.5022e-25, 3.7459e-27, 2.8567e-26, 7.4228e-26, 4.1649e-25,\n 3.9418e-26, 5.1269e-26, 1.7870e-25, 4.6470e-26, 1.8847e-25, 2.0403e-25,\n 4.0492e-26, 1.5928e-26, 1.3856e-25, 4.9813e-26, 1.0225e-25, 4.7457e-25,\n 3.2807e-26, 6.7568e-26, 5.3305e-26, 1.9272e-25, 3.4215e-25, 3.4854e-25,\n 1.3613e-25, 1.5222e-26, 2.7525e-26, 4.0938e-26, 1.3827e-26, 1.1391e-26,\n 1.5174e-26, 4.7029e-26, 3.1517e-27, 5.1075e-26, 1.8332e-26, 1.5739e-26,\n 3.1316e-27, 2.5454e-26, 8.3901e-27, 3.4926e-26, 1.9645e-26, 6.0496e-26,\n 1.4194e-26, 2.8116e-26, 6.8974e-27, 6.4241e-27, 2.2950e-27, 5.6124e-27,\n 1.6909e-26, 3.7951e-27, 3.2557e-26, 2.2904e-26, 6.8892e-26, 1.3499e-26,\n 7.4963e-26, 5.8642e-27, 6.9667e-27, 1.4910e-26, 1.0710e-26, 2.4347e-25,\n 1.5320e-26, 4.6714e-26, 3.7380e-26, 1.3868e-25, 3.1305e-26, 1.3536e-26,\n 4.4720e-26, 8.7951e-26, 1.6641e-26, 7.2820e-27, 1.6529e-25, 5.8390e-27,\n 1.7915e-26, 2.6550e-26, 1.8652e-26, 1.1244e-25, 8.5775e-26, 1.0779e-25,\n 9.0600e-27, 2.8772e-26, 5.8116e-27, 2.3106e-26, 2.0127e-26, 6.2827e-26,\n 3.4154e-26, 6.4942e-26, 2.8857e-26, 2.8776e-26, 2.9997e-26, 7.3039e-27,\n 1.2257e-25, 2.3030e-25, 1.2771e-25, 4.8051e-26, 2.7413e-25, 2.4133e-25,\n 9.4016e-26, 1.0886e-25, 3.8051e-26, 1.9144e-25, 1.5001e-25, 7.5727e-26,\n 5.5348e-26, 8.3590e-26, 1.0634e-25, 2.0488e-26, 1.2210e-25, 4.8183e-26,\n 7.3740e-26, 2.4241e-26, 3.0028e-26, 1.9968e-25, 6.8984e-26, 3.1429e-25,\n 4.8058e-26, 1.9629e-26, 1.4020e-25, 1.7312e-25, 1.5648e-25, 5.5990e-26,\n 1.6561e-26, 1.5814e-25, 2.5822e-07, 4.0349e-08, 1.0900e-07, 4.6772e-08,\n 6.5725e-07, 5.4317e-08, 4.3984e-07, 6.7397e-08, 1.1898e-07, 6.7019e-09,\n 8.3552e-09, 1.2919e-07, 1.2338e-07, 1.6983e-08, 4.6764e-08, 5.1275e-07,\n 1.5497e-07, 4.0088e-08, 2.8788e-07, 6.2537e-08, 2.8590e-07, 4.3416e-08,\n 7.1565e-08, 1.6321e-08, 3.6387e-08, 3.7443e-08, 4.0968e-07, 4.5842e-07,\n 3.0185e-07, 3.1675e-07, 1.1180e-07, 6.0315e-08, 9.4869e-08, 1.4437e-07,\n 2.2604e-07, 7.6199e-09, 9.4643e-07, 9.9454e-08, 1.6883e-08, 1.8661e-07,\n 3.5643e-08, 2.1013e-07, 4.5577e-07, 5.3570e-07, 2.4347e-07, 2.2290e-08,\n 6.4482e-07, 1.4947e-07, 2.0517e-07, 4.6845e-08, 4.6279e-08, 2.9403e-07,\n 5.6857e-07, 2.5884e-07, 1.0459e-07, 8.4439e-08, 1.4792e-07, 2.8235e-08,\n 5.3007e-08, 6.0947e-08, 9.3598e-07, 9.3101e-08, 6.7333e-08, 9.7504e-09,\n 1.0678e-07, 1.0459e-08, 1.9096e-07, 4.5527e-08, 2.3156e-07, 9.6150e-07,\n 4.2125e-08, 3.0159e-07, 4.1959e-07, 6.8573e-08, 4.2664e-07, 5.2500e-08,\n 5.6987e-09, 1.5000e-08, 4.2949e-09, 6.5858e-08, 6.1123e-08, 4.8943e-07,\n 2.4661e-08, 2.9873e-07, 1.0272e-07, 5.2668e-08, 7.4968e-08, 5.4779e-09,\n 1.2544e-07, 7.1154e-08, 6.4591e-08, 3.9116e-07, 5.6215e-08, 5.3114e-07,\n 4.6023e-07, 4.0838e-08, 1.6591e-07, 5.2559e-07, 4.0526e-07, 1.5609e-07,\n 2.3065e-09, 1.7307e-07, 4.5737e-08, 2.9083e-07, 7.6063e-07, 2.1594e-08,\n 1.2301e-08, 4.3314e-08, 2.4759e-08, 4.6293e-08, 8.3343e-07, 3.3151e-08,\n 1.6002e-08, 8.5757e-08, 1.6752e-07, 1.8418e-07, 4.5536e-07, 3.4873e-07,\n 5.2284e-08, 4.9782e-08, 4.8347e-07, 1.2873e-08, 5.5029e-07, 2.1415e-08,\n 4.3837e-08, 1.9844e-07, 3.9213e-08, 7.5763e-09, 4.0781e-07, 1.0202e-07,\n 2.5371e-07, 3.3030e-07, 1.2978e-08, 1.6427e-07, 1.4889e-08, 1.2601e-07,\n 9.0208e-08, 4.1749e-07, 3.2014e-07, 2.1445e-08, 3.5973e-07, 1.6146e-07,\n 1.7125e-08, 2.3912e-08, 3.9018e-07, 2.0475e-07, 3.5096e-08, 1.0306e-07,\n 1.6516e-07, 5.2548e-08, 1.1184e-08, 1.9830e-08, 5.4469e-07, 1.0635e-08,\n 1.2112e-07, 3.0481e-08, 4.7060e-07, 1.1921e-07, 4.4845e-08, 2.7197e-07,\n 1.7456e-07, 6.1314e-08, 3.1906e-07, 1.6881e-08, 2.6802e-07, 1.2065e-08,\n 5.0210e-08, 6.6482e-07, 3.5814e-08, 1.8693e-07, 3.5000e-08, 1.5323e-07,\n 8.5227e-08, 1.1770e-07, 1.9349e-07, 4.8561e-08, 2.6576e-07, 6.4653e-08,\n 4.1196e-08, 6.7655e-08, 3.0584e-08, 7.7150e-09, 8.5459e-10, 6.9525e-08,\n 1.6736e-07, 3.0557e-09, 1.3484e-07, 3.8636e-08, 2.6487e-07, 1.3528e-07,\n 2.2983e-07, 6.6785e-08, 5.8502e-08, 1.4382e-07, 3.2013e-08, 2.5610e-07,\n 2.6431e-08, 8.3995e-08, 4.6868e-09, 3.7396e-07, 2.9136e-08, 1.0525e-07,\n 2.1047e-07, 1.1485e-07, 1.5429e-07, 2.3460e-07, 3.1481e-07, 1.9422e-07,\n 1.1199e-07, 7.1852e-08, 1.0283e-07, 2.4540e-07, 9.0040e-07, 1.5030e-07,\n 9.1842e-08, 3.7065e-07, 1.7980e-07, 1.0970e-07, 1.6542e-07, 4.2332e-08,\n 6.9785e-07, 3.6421e-08, 1.7375e-07, 5.1310e-07, 1.5205e-07, 4.6574e-09,\n 9.1309e-08, 7.9846e-08, 8.3148e-07, 1.9617e-07, 5.0383e-09, 1.9093e-07,\n 4.6075e-07, 2.9644e-07, 1.7023e-07, 2.8652e-07, 4.1797e-08, 9.1271e-07,\n 2.5687e-08, 4.3898e-07, 1.0301e-07, 5.4093e-08, 1.1564e-08, 1.1274e-06,\n 9.5188e-08, 1.0351e-07, 3.2613e-07, 1.6061e-07, 4.1993e-07, 2.1980e-08,\n 3.7727e-07, 2.3076e-07, 3.3954e-07, 1.2695e-08, 4.1313e-07, 1.4179e-07],\n device='cuda:0')" + }, + "54": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-4.9490e-17, 4.2564e-17, -2.5723e-17, ..., -3.5677e-17,\n -6.9006e-17, 5.8334e-17],\n [-1.7512e-17, 1.3654e-17, -7.8447e-18, ..., -9.7562e-18,\n -1.5351e-17, 2.1218e-17],\n [ 4.7955e-17, -3.9872e-17, 2.0746e-17, ..., 2.9544e-17,\n 5.2771e-17, -5.3427e-17],\n ...,\n [ 5.3947e-17, -4.6068e-17, 2.4600e-17, ..., 3.7218e-17,\n 7.1139e-17, -5.8444e-17],\n [ 7.6977e-17, -6.7145e-17, 4.2403e-17, ..., 5.4405e-17,\n 9.8798e-17, -9.6285e-17],\n [ 1.9866e-17, -1.8116e-17, 1.0936e-17, ..., 1.2857e-17,\n 1.8860e-17, -2.4980e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[4.0153e-09, 2.4600e-08, 6.6132e-08, ..., 3.3512e-10, 3.8312e-08,\n 4.2919e-08],\n [4.6840e-09, 3.5359e-08, 7.9546e-08, ..., 5.1060e-10, 2.4747e-08,\n 4.1313e-08],\n [3.3191e-09, 1.3521e-08, 1.9591e-08, ..., 3.1846e-10, 1.1351e-08,\n 1.6754e-08],\n ...,\n [5.3076e-09, 2.6077e-08, 9.2349e-08, ..., 6.2177e-10, 6.5934e-08,\n 5.1222e-08],\n [7.6812e-10, 5.0133e-09, 1.3397e-08, ..., 9.1634e-11, 1.0836e-08,\n 1.2808e-08],\n [3.5169e-09, 1.7778e-08, 5.0962e-08, ..., 9.0289e-11, 1.7413e-08,\n 2.3729e-08]], device='cuda:0')" + }, + "55": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-7.0400e-17, -2.0782e-17, 6.0771e-17, 3.0851e-18, 7.5753e-17,\n -3.4378e-17, 4.9988e-17, -5.3207e-17, 1.0370e-16, -2.0036e-17,\n 5.5812e-18, -4.9100e-17, -6.6435e-17, -1.7570e-17, -1.1090e-16,\n -7.8004e-17, 2.1017e-17, -6.7852e-17, -8.2414e-18, 4.2371e-17,\n -8.9119e-17, -2.7911e-17, -1.1210e-16, 2.6307e-17, -8.5124e-17,\n -5.3632e-17, 4.0010e-17, -1.5556e-17, 3.5828e-17, 7.2113e-17,\n 7.6401e-17, 4.7439e-17, 5.9347e-17, -4.4800e-17, 8.3368e-17,\n 6.6112e-17, 1.1380e-16, 8.0858e-17, 2.1979e-17, 8.3961e-17,\n 3.4881e-17, 6.3797e-17, 3.6272e-17, -6.3376e-17, 7.6211e-17,\n -5.9902e-17, 2.8688e-17, -7.7176e-17, -2.2987e-18, 1.0967e-17,\n -7.3753e-17, -4.7141e-18, 6.1472e-17, 2.4170e-17, 1.1531e-16,\n -1.0058e-16, -1.5578e-17, 6.9276e-17, -7.3440e-17, 7.7239e-17,\n 8.0880e-17, -6.6871e-17, 3.0262e-17, 5.8644e-17, -7.3344e-17,\n -5.0635e-17, -1.0189e-16, -6.1527e-17, -4.8377e-17, -2.3126e-17,\n -9.6360e-17, 1.0267e-16, 8.0030e-17, 5.9554e-17, -6.1854e-17,\n -6.7695e-17, -6.6913e-17, -4.4008e-17, -7.9726e-17, 1.8740e-17,\n -8.8695e-17, -1.3683e-17, -4.9607e-17, 2.7765e-17, 1.0057e-16,\n -2.7767e-17, 1.8505e-17, 6.5043e-17, -6.9251e-18, -6.1678e-17,\n 4.2633e-17, 7.9637e-17, -3.1888e-18, -8.4883e-17, 5.3502e-17,\n -5.6435e-17, 3.8909e-17, -1.7585e-17, 9.4424e-17, 4.0451e-17,\n 9.8355e-18, -5.3698e-17, -5.3850e-17, 7.6113e-17, 9.6236e-17,\n 5.5403e-17, 5.7624e-17, -1.5057e-17, -4.0373e-17, 1.0348e-16,\n -4.4848e-17, -9.0035e-17, 8.7554e-17, -4.3867e-17, 1.1137e-17,\n -5.4477e-17, 3.3400e-17, -4.1142e-17, 6.0243e-17, 5.1220e-17,\n -3.9261e-17, -2.8384e-17, -7.0458e-17, 3.4565e-17, -1.0395e-16,\n -3.6588e-17, 1.0666e-16, -2.6387e-17, -7.4749e-17, -2.6032e-17,\n 8.6271e-17, -5.5329e-17, -8.0326e-17, -6.8582e-17, -7.9834e-17,\n 8.9778e-17, 5.8457e-17, -3.5602e-17, 5.1037e-17, 5.2738e-17,\n -1.0573e-16, 8.1111e-17, -4.9853e-17, -1.1495e-16, -4.3093e-17,\n 2.8228e-17, -2.7541e-17, 4.7611e-17, 1.0493e-16, 7.9578e-17,\n 2.6344e-17, -1.2902e-17, -9.0670e-17, 5.6852e-17, 8.8452e-17,\n 8.3382e-18, 5.4763e-17, -8.6283e-17, -2.9847e-17, 4.5014e-17,\n 5.8426e-17, -5.7472e-17, -2.5569e-17, 1.0015e-16, 6.0665e-19,\n 3.4248e-17, -9.5497e-17, 8.4043e-17, -1.1383e-18, 1.0805e-16,\n -9.3214e-18, 3.7583e-18, -1.5567e-16, 7.5779e-17, -8.0315e-17,\n 1.4795e-16, 5.7341e-17, 7.2150e-17, 2.5878e-17, -1.0444e-16,\n -4.6975e-17, -5.3904e-17, 7.2795e-17, 1.6252e-17, -5.3412e-17,\n 3.2874e-17, -1.1948e-16, 5.9751e-17, 5.8577e-17, -9.3890e-17,\n -7.7895e-17, -1.0193e-16, 2.0472e-17, -8.8074e-17, -7.3662e-17,\n 4.6673e-17, -1.8517e-17, 2.9791e-17, 7.1933e-17, 3.5849e-17,\n 3.6800e-17, 7.9365e-17, 9.6909e-17, -1.2710e-16, 8.2146e-17,\n 3.4955e-17, 7.0418e-17, 1.0582e-16, 8.9840e-17, 2.6177e-17,\n -7.3014e-17, -4.7089e-17, 5.1320e-17, 8.7017e-17, 6.2042e-17,\n 2.9890e-17, -9.5138e-17, 8.9125e-17, -3.3959e-17, 8.2917e-17,\n -2.8556e-17, 1.2303e-16, 5.0355e-17, 4.8294e-17, -5.2993e-17,\n 3.2219e-17, -7.7568e-17, -1.1550e-16, 7.4818e-17, 9.5545e-17,\n 3.0543e-17, -7.4894e-17, -7.5912e-17, -8.3200e-17, 8.4145e-17,\n -7.9320e-17, 4.3136e-17, 1.5458e-18, -7.6985e-17, -6.8364e-17,\n -7.9931e-17, -2.9322e-17, -4.9044e-18, -5.4810e-17, -6.6232e-17,\n -7.4272e-18, -4.8633e-17, 3.3260e-17, 5.6414e-17, 6.7534e-17,\n 5.0597e-17, 1.2096e-16, -2.3056e-17, 7.3325e-17, 1.1093e-16,\n 2.8678e-17], device='cuda:0')", + "exp_avg_sq": "tensor([3.3307e-07, 4.4912e-07, 1.0950e-07, 1.8649e-06, 2.3567e-06, 8.3516e-08,\n 1.0355e-07, 2.5276e-08, 1.1670e-07, 1.3429e-08, 2.5947e-08, 1.0258e-07,\n 1.3992e-07, 5.0210e-07, 1.1722e-07, 2.6812e-08, 5.6664e-07, 5.8771e-07,\n 2.4511e-07, 3.5120e-07, 2.1129e-06, 1.4259e-06, 2.5958e-07, 1.0942e-06,\n 5.6097e-07, 1.3675e-06, 1.4070e-08, 8.0224e-07, 6.4158e-08, 6.8461e-08,\n 2.2085e-07, 1.4978e-07, 7.7092e-08, 7.7527e-08, 1.4404e-06, 8.3205e-08,\n 3.9338e-06, 3.5819e-07, 5.4351e-07, 2.1395e-07, 5.2139e-08, 1.0615e-07,\n 9.2632e-08, 4.1367e-07, 6.0905e-07, 9.7291e-07, 4.0711e-08, 2.4984e-07,\n 5.6568e-08, 1.1725e-07, 2.2220e-07, 6.1891e-08, 6.8218e-07, 9.3933e-08,\n 2.6383e-06, 2.4124e-07, 1.1710e-07, 1.7363e-06, 9.8085e-07, 2.0355e-08,\n 1.8683e-06, 1.0988e-06, 1.1811e-06, 3.6189e-07, 2.1974e-07, 1.4637e-06,\n 1.3482e-06, 2.0535e-08, 2.6008e-08, 1.1673e-06, 1.8359e-06, 2.6649e-07,\n 3.4387e-07, 6.0123e-08, 9.9705e-08, 9.7245e-07, 2.4497e-08, 3.2212e-07,\n 9.9249e-07, 6.1215e-08, 1.5476e-07, 1.1016e-07, 5.5266e-07, 2.3691e-07,\n 2.7325e-07, 6.1126e-08, 1.5606e-06, 1.9864e-08, 2.3239e-07, 1.1797e-07,\n 8.1513e-08, 4.8576e-07, 1.9360e-06, 2.7076e-07, 2.5315e-08, 2.2196e-07,\n 8.2024e-08, 1.9635e-07, 4.8233e-07, 1.5744e-08, 5.6882e-07, 2.5799e-08,\n 2.1835e-07, 9.2465e-07, 1.1774e-07, 2.4480e-07, 2.6081e-08, 2.4140e-07,\n 6.4463e-07, 1.7671e-07, 2.9858e-07, 1.9383e-07, 1.4280e-07, 1.7823e-08,\n 8.3616e-08, 1.3295e-07, 1.5824e-07, 5.2498e-07, 4.4803e-07, 2.8605e-07,\n 3.8020e-07, 1.0911e-07, 2.7023e-07, 6.3131e-08, 1.8061e-06, 1.2249e-07,\n 8.7220e-08, 1.0325e-07, 4.3538e-07, 4.8364e-07, 4.6809e-07, 1.4042e-06,\n 1.9162e-08, 1.0370e-06, 9.3147e-07, 8.3006e-08, 7.6135e-07, 1.0537e-07,\n 1.9374e-07, 3.4618e-07, 2.5940e-07, 3.6352e-07, 6.9007e-07, 9.2042e-07,\n 3.8063e-08, 1.7969e-07, 5.5846e-08, 6.7551e-09, 3.5221e-08, 1.2412e-07,\n 1.4376e-08, 1.3821e-06, 1.7854e-07, 3.8730e-07, 1.4101e-06, 2.6108e-07,\n 3.2761e-07, 1.7565e-06, 1.1577e-08, 9.3953e-08, 1.1628e-07, 4.0256e-08,\n 1.1546e-07, 1.1680e-06, 8.2851e-07, 4.0013e-07, 6.3770e-07, 9.9611e-07,\n 1.0275e-07, 2.7419e-07, 2.4122e-08, 3.1542e-07, 5.3986e-07, 2.5201e-07,\n 3.8942e-07, 3.3252e-07, 2.5870e-08, 5.1585e-07, 4.3580e-07, 2.4655e-08,\n 3.0034e-07, 1.8485e-07, 5.7532e-08, 1.0209e-06, 7.7635e-08, 1.8341e-06,\n 7.7114e-08, 2.3129e-08, 5.0090e-08, 3.8509e-08, 7.1663e-07, 5.0406e-07,\n 2.5545e-07, 2.1302e-07, 1.4860e-06, 2.0006e-07, 3.7729e-07, 1.3028e-08,\n 1.7054e-06, 1.0192e-07, 7.0416e-07, 2.6089e-07, 5.0418e-07, 3.0738e-07,\n 2.6232e-07, 1.9517e-08, 6.5124e-08, 8.2922e-07, 4.6829e-07, 1.2240e-06,\n 1.1201e-06, 1.2102e-06, 1.7544e-07, 3.8966e-07, 1.7885e-07, 1.3104e-06,\n 8.5025e-07, 8.0689e-07, 1.1739e-06, 2.1352e-08, 3.5721e-08, 6.0038e-08,\n 3.5322e-07, 1.2311e-06, 4.5509e-07, 9.7520e-08, 2.3829e-07, 1.7545e-06,\n 2.2566e-07, 7.3425e-07, 3.8529e-08, 5.7369e-07, 8.6024e-07, 6.5149e-08,\n 1.2169e-07, 2.5421e-07, 3.6278e-07, 1.1608e-08, 5.7272e-07, 1.9128e-08,\n 4.2399e-08, 1.2724e-07, 5.9871e-07, 2.4059e-08, 4.1647e-07, 1.2841e-07,\n 1.4645e-08, 1.7983e-08, 9.4304e-07, 2.1113e-08, 2.3266e-07, 8.6837e-07,\n 1.2942e-07, 4.7919e-07, 6.3719e-08, 2.6866e-07], device='cuda:0')" + }, + "56": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-4.2488e-17, -7.7212e-17, 7.0804e-17, ..., 1.3247e-16,\n 7.7171e-17, -5.7171e-17],\n [ 7.1613e-17, 1.2058e-16, -9.7651e-17, ..., -1.9023e-16,\n -1.1755e-16, 7.4390e-17],\n ...,\n [ 8.0942e-17, 1.3547e-16, -1.0649e-16, ..., -2.0632e-16,\n -1.2757e-16, 7.9687e-17],\n [ 3.2899e-16, 5.4936e-16, -4.3668e-16, ..., -8.5442e-16,\n -5.2986e-16, 3.3194e-16],\n [ 9.6982e-17, 1.6098e-16, -1.2789e-16, ..., -2.4881e-16,\n -1.5447e-16, 9.6445e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.2816e-07, 1.2343e-06, 3.0888e-06, ..., 1.0873e-05, 4.8949e-06,\n 7.3916e-07],\n [4.1203e-09, 1.7707e-08, 3.8162e-08, ..., 1.3403e-07, 6.0073e-08,\n 1.0060e-08],\n [3.7814e-09, 1.3796e-08, 3.5491e-08, ..., 1.2579e-07, 5.6519e-08,\n 8.2731e-09],\n ...,\n [4.6706e-09, 1.7853e-08, 4.6034e-08, ..., 1.5816e-07, 7.2436e-08,\n 1.1472e-08],\n [7.3134e-09, 2.5182e-08, 5.4350e-08, ..., 2.2813e-07, 9.2162e-08,\n 1.0179e-08],\n [3.8530e-09, 1.3441e-08, 3.3520e-08, ..., 1.2282e-07, 5.4044e-08,\n 7.4350e-09]], device='cuda:0')" + }, + "57": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.6052e-45, -5.9026e-17, 8.6160e-17, 4.5005e-17, 4.8607e-17,\n 8.3060e-17, 9.8882e-17, 9.3892e-17, 3.8534e-16, 1.1280e-16],\n device='cuda:0')", + "exp_avg_sq": "tensor([9.6817e-05, 1.1631e-06, 1.1191e-06, 8.8050e-07, 1.0579e-06, 9.8182e-07,\n 1.6459e-06, 1.4720e-06, 1.5901e-06, 1.0353e-06], device='cuda:0')" + }, + "58": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-4.2018e-17, -7.6358e-17, 7.0021e-17, ..., 1.3100e-16,\n 7.6317e-17, -5.6539e-17],\n [ 7.0821e-17, 1.1925e-16, -9.6570e-17, ..., -1.8813e-16,\n -1.1625e-16, 7.3566e-17],\n ...,\n [ 8.0047e-17, 1.3397e-16, -1.0531e-16, ..., -2.0404e-16,\n -1.2616e-16, 7.8805e-17],\n [ 3.2535e-16, 5.4328e-16, -4.3185e-16, ..., -8.4496e-16,\n -5.2400e-16, 3.2826e-16],\n [ 9.5909e-17, 1.5919e-16, -1.2647e-16, ..., -2.4605e-16,\n -1.5276e-16, 9.5378e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.2647e-07, 1.2292e-06, 3.0796e-06, ..., 1.0825e-05, 4.8778e-06,\n 7.3833e-07],\n [4.0977e-09, 1.7636e-08, 3.8050e-08, ..., 1.3339e-07, 5.9856e-08,\n 1.0049e-08],\n [3.7622e-09, 1.3740e-08, 3.5385e-08, ..., 1.2523e-07, 5.6322e-08,\n 8.2634e-09],\n ...,\n [4.6472e-09, 1.7783e-08, 4.5903e-08, ..., 1.5747e-07, 7.2193e-08,\n 1.1460e-08],\n [7.2733e-09, 2.5052e-08, 5.4146e-08, ..., 2.2699e-07, 9.1774e-08,\n 1.0160e-08],\n [3.8332e-09, 1.3383e-08, 3.3415e-08, ..., 1.2226e-07, 5.3846e-08,\n 7.4254e-09]], device='cuda:0')" + }, + "59": { + "step": "tensor(1252.)", + "exp_avg": "tensor([-5.6052e-45, -5.8373e-17, 8.5206e-17, 4.4507e-17, 4.8069e-17,\n 8.2141e-17, 9.7787e-17, 9.2853e-17, 3.8107e-16, 1.1155e-16],\n device='cuda:0')", + "exp_avg_sq": "tensor([9.6578e-05, 1.1604e-06, 1.1163e-06, 8.7856e-07, 1.0556e-06, 9.7942e-07,\n 1.6421e-06, 1.4685e-06, 1.5851e-06, 1.0326e-06], device='cuda:0')" + }, + "60": { + "step": "tensor(1252.)", + "exp_avg": "tensor([[-5.6052e-45, -5.6052e-45, 5.6052e-45, ..., 5.6052e-45,\n 5.6052e-45, 5.6052e-45],\n [-4.2549e-17, -7.7324e-17, 7.0906e-17, ..., 1.3266e-16,\n 7.7282e-17, -5.7254e-17],\n [ 7.1717e-17, 1.2076e-16, -9.7792e-17, ..., -1.9050e-16,\n -1.1772e-16, 7.4497e-17],\n ...,\n [ 8.1059e-17, 1.3566e-16, -1.0665e-16, ..., -2.0662e-16,\n -1.2775e-16, 7.9802e-17],\n [ 3.2947e-16, 5.5015e-16, -4.3731e-16, ..., -8.5565e-16,\n -5.3063e-16, 3.3242e-16],\n [ 9.7122e-17, 1.6121e-16, -1.2807e-16, ..., -2.4916e-16,\n -1.5469e-16, 9.6584e-17]], device='cuda:0')", + "exp_avg_sq": "tensor([[3.2817e-07, 1.2343e-06, 3.0888e-06, ..., 1.0874e-05, 4.8950e-06,\n 7.3917e-07],\n [4.1206e-09, 1.7709e-08, 3.8163e-08, ..., 1.3403e-07, 6.0075e-08,\n 1.0060e-08],\n [3.7816e-09, 1.3797e-08, 3.5492e-08, ..., 1.2579e-07, 5.6520e-08,\n 8.2732e-09],\n ...,\n [4.6708e-09, 1.7854e-08, 4.6035e-08, ..., 1.5816e-07, 7.2438e-08,\n 1.1472e-08],\n [7.3139e-09, 2.5184e-08, 5.4351e-08, ..., 2.2814e-07, 9.2165e-08,\n 1.0180e-08],\n [3.8532e-09, 1.3442e-08, 3.3521e-08, ..., 1.2283e-07, 5.4045e-08,\n 7.4351e-09]], device='cuda:0')" + }, + "61": { "step": "tensor(1252.)", - "exp_avg": "tensor([[ 3.9230e-04, 8.6013e-05, -4.0562e-28, ..., -8.0357e-06,\n 3.1746e-05, -1.4099e-05],\n [ 4.2048e-04, 1.7068e-04, -2.2865e-28, ..., -1.9681e-05,\n -7.8058e-05, -9.3920e-05],\n [-5.8150e-04, -5.1838e-05, 1.9939e-28, ..., 3.0194e-07,\n 1.7539e-04, -8.2012e-06],\n ...,\n [ 3.6278e-04, 3.8798e-05, 4.8383e-29, ..., -1.4735e-05,\n -9.2320e-06, 1.6069e-05],\n [-3.5278e-04, -2.1873e-04, -1.6439e-28, ..., 3.7640e-05,\n -1.1982e-04, 3.7282e-05],\n [-6.4497e-05, 2.1634e-04, -2.8426e-28, ..., 5.2144e-06,\n 1.2079e-05, 1.8201e-05]], device='cuda:0')", - "exp_avg_sq": "tensor([[1.0953e-06, 1.3442e-06, 1.1048e-10, ..., 3.9482e-09, 8.9938e-09,\n 1.8019e-08],\n [2.1084e-06, 2.6551e-06, 1.6841e-10, ..., 1.5030e-09, 2.3215e-08,\n 9.7827e-08],\n [1.8753e-06, 2.5430e-06, 4.6574e-10, ..., 2.7687e-09, 8.8683e-09,\n 6.6008e-08],\n ...,\n [1.7657e-06, 2.8547e-06, 1.5489e-10, ..., 2.8203e-09, 1.1983e-08,\n 6.1273e-08],\n [1.7073e-06, 2.9791e-06, 1.0857e-10, ..., 2.7219e-09, 1.2247e-08,\n 6.0122e-08],\n [2.1537e-06, 3.3493e-06, 1.6559e-10, ..., 2.1838e-09, 2.0388e-08,\n 3.8560e-08]], device='cuda:0')" + "exp_avg": "tensor([-5.6052e-45, -5.9111e-17, 8.6284e-17, 4.5070e-17, 4.8677e-17,\n 8.3180e-17, 9.9024e-17, 9.4027e-17, 3.8589e-16, 1.1296e-16],\n device='cuda:0')", + "exp_avg_sq": "tensor([9.6817e-05, 1.1632e-06, 1.1191e-06, 8.8051e-07, 1.0579e-06, 9.8183e-07,\n 1.6459e-06, 1.4720e-06, 1.5901e-06, 1.0353e-06], device='cuda:0')" } }, "param_groups": [ { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "shared", "betas": [ 0.9, @@ -52,7 +212,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_256", "betas": [ 0.9, @@ -75,7 +235,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_512", "betas": [ 0.9, @@ -98,7 +258,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_768", "betas": [ 0.9, @@ -121,7 +281,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_1024", "betas": [ 0.9, @@ -144,7 +304,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_1280", "betas": [ 0.9, @@ -167,7 +327,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_1536", "betas": [ 0.9, @@ -190,7 +350,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_1792", "betas": [ 0.9, @@ -213,7 +373,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_2048", "betas": [ 0.9, @@ -236,7 +396,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_2304", "betas": [ 0.9, @@ -259,7 +419,7 @@ ] }, { - "lr": 0.0009755527298894294, + "lr": 0.0009046039886902864, "name": "scale_2560", "betas": [ 0.9, @@ -282,7 +442,7 @@ ] }, { - "lr": 0.00048778860081564085, + "lr": 0.0004523497400965494, "name": "fusion", "betas": [ 0.9, @@ -338,7 +498,7 @@ "T_i": 10, "T_mult": 2, "eta_min": 1e-06, - "T_cur": 1, + "T_cur": 2, "base_lrs": [ 0.001, 0.001, @@ -353,30 +513,31 @@ 0.001, 0.0005 ], - "last_epoch": 1, + "last_epoch": 2, "_step_count": 0, "_is_initial": false, "_get_lr_called_within_step": false, "_last_lr": [ - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.0009755527298894294, - 0.00048778860081564085 + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0009046039886902864, + 0.0004523497400965494 ] }, "metrics": { - "best_val_acc": 80.786, - "best_epoch": 0, + "best_val_acc": 81.526, + "best_epoch": 1, "scale_accuracies": { - "256": 80.786 + "256": 81.526, + "512": 81.2 } }, "train_config": {