Upload weights and configs - Run 20251012_032356

weights/best_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc187ab3f8538420f8a3440dd736f259e0c66a6587c78a5bbd13699917e2d8d9
+oid sha256:e3b0c800988cf782b1b9980bc9ac31f9de8953dc7f0e37707281a8e81c83c521
 size 59515088

weights/best_model_metadata.json

@@ -1,26 +1,26 @@
 {
-  "epoch": 1,
+  "epoch": 2,
   "optimizer_state_dict": {
     "state": {
       "0": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([[-2.9847e-05, -1.5561e-05, -3.2015e-05,  ..., -3.4633e-05,\n          9.6170e-05, -4.8952e-05],\n        [-5.1054e-05, -3.7014e-05,  2.2638e-05,  ..., -1.1228e-04,\n         -1.0936e-04,  3.1119e-05],\n        [ 2.6958e-06, -1.2456e-04,  1.4858e-05,  ..., -4.4155e-05,\n          6.4120e-05,  2.8139e-05],\n        ...,\n        [-5.4542e-05,  2.4455e-05, -1.6261e-05,  ..., -3.6784e-05,\n          4.9569e-05, -5.8800e-05],\n        [-4.4965e-23,  2.3582e-24,  2.9657e-23,  ..., -1.0940e-23,\n          6.9415e-24, -2.3902e-23],\n        [-5.8117e-05,  2.1739e-05, -4.2380e-05,  ..., -5.3147e-05,\n          3.0465e-05,  8.0894e-05]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[8.1019e-08, 1.0675e-07, 1.2200e-08,  ..., 2.7520e-08, 2.9280e-08,\n         8.1349e-09],\n        [2.0122e-07, 1.4826e-07, 4.4990e-08,  ..., 8.7383e-08, 4.9043e-08,\n         6.1986e-08],\n        [1.6635e-07, 2.2215e-07, 4.7403e-08,  ..., 5.8297e-08, 3.8775e-08,\n         4.2303e-08],\n        ...,\n        [4.0419e-07, 2.7279e-07, 4.6405e-08,  ..., 4.8686e-08, 4.1250e-08,\n         2.8963e-08],\n        [6.1337e-11, 1.4214e-09, 2.2304e-10,  ..., 9.4234e-11, 2.9866e-10,\n         1.7780e-10],\n        [2.7880e-07, 1.7346e-07, 2.7628e-08,  ..., 3.3208e-08, 6.5641e-08,\n         3.2027e-08]], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[ 1.7831e-05, -4.5677e-05, -1.0656e-05,  ...,  2.5515e-05,\n          2.5247e-05,  2.0031e-05],\n        [ 2.8097e-05, -8.2606e-05,  2.0928e-05,  ...,  5.4769e-06,\n          6.5177e-06,  1.4157e-05],\n        [-1.5467e-05, -4.2629e-05, -4.7339e-06,  ...,  5.2465e-05,\n          1.1511e-05,  1.5347e-05],\n        ...,\n        [-2.3319e-04,  1.3111e-04, -6.7066e-05,  ..., -8.4160e-05,\n         -1.3645e-05, -5.2447e-05],\n        [ 2.8317e-32, -3.0252e-32,  3.7288e-32,  ...,  1.2018e-33,\n         -2.5549e-32, -1.2403e-32],\n        [ 2.8868e-04, -2.0421e-04,  5.1263e-05,  ...,  1.0802e-04,\n          6.9886e-05, -1.3845e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[5.5924e-08, 7.8008e-08, 9.2438e-09,  ..., 2.2459e-08, 2.2992e-08,\n         6.1665e-09],\n        [1.5358e-07, 1.1598e-07, 3.8675e-08,  ..., 7.3238e-08, 3.6567e-08,\n         5.1830e-08],\n        [1.1599e-07, 1.4503e-07, 3.3933e-08,  ..., 4.0283e-08, 2.8559e-08,\n         3.0028e-08],\n        ...,\n        [3.0393e-07, 1.8733e-07, 3.2085e-08,  ..., 3.6319e-08, 2.9407e-08,\n         1.9908e-08],\n        [1.7530e-11, 4.0619e-10, 6.3743e-11,  ..., 2.6928e-11, 8.5347e-11,\n         5.0808e-11],\n        [1.9579e-07, 1.2667e-07, 1.9311e-08,  ..., 2.4558e-08, 4.5792e-08,\n         2.2544e-08]], device='cuda:0')"
       },
       "1": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([ 1.7719e-03, -2.7156e-03,  1.7750e-03,  6.0193e-04, -1.2922e-15,\n         2.1121e-04,  4.9933e-04, -2.6053e-03,  3.6154e-04,  7.6070e-04,\n         2.2209e-04, -3.5991e-03,  5.6052e-45,  2.6509e-04,  5.7084e-04,\n         1.4721e-05, -4.8943e-04,  4.4611e-03,  7.7828e-04,  5.6052e-45,\n         1.3613e-04, -3.5591e-04,  3.9319e-04,  5.0281e-04,  5.6052e-45,\n         5.8405e-04, -2.1400e-04,  2.3399e-03,  3.3233e-03, -1.1443e-03,\n        -1.8175e-03,  1.7404e-03, -6.8076e-11,  5.6052e-45, -9.1142e-04,\n         1.8191e-04, -2.5487e-03,  2.2662e-03,  1.0145e-03, -1.1766e-03,\n        -2.9495e-04,  3.0875e-03,  1.2050e-04, -6.2883e-03, -2.0681e-04,\n        -1.0954e-03, -2.4520e-03, -7.5239e-05,  1.6260e-03,  1.2031e-03,\n        -8.9359e-04,  5.6052e-45,  4.0436e-08,  5.6052e-45,  9.4041e-04,\n        -8.9957e-03,  8.6776e-04,  1.6575e-03,  5.6052e-45,  2.1743e-03,\n        -2.7675e-03, -1.0951e-03, -1.4413e-03,  2.6270e-03,  3.0545e-03,\n        -5.3882e-04, -2.0457e-03,  5.6052e-45, -1.2317e-03, -2.6512e-03,\n        -1.2740e-04,  5.6052e-45,  3.3783e-03,  8.1476e-04,  7.9997e-04,\n        -1.2763e-05,  2.0720e-04, -1.9931e-03, -2.9498e-03, -4.0966e-05,\n         4.4024e-04,  2.7361e-04, -1.2984e-03,  2.0401e-03,  2.5797e-03,\n        -1.8919e-04, -2.6561e-03, -2.7024e-03,  6.2633e-04,  2.1254e-03,\n         5.6052e-45, -3.3968e-03, -1.4859e-03,  3.0709e-04,  3.1062e-04,\n        -1.3830e-03, -1.2511e-03,  1.5813e-04,  2.0279e-03,  1.6878e-03,\n         2.7028e-03, -1.5681e-05,  1.8112e-03,  2.0162e-03,  1.1955e-03,\n         3.8765e-04, -1.6990e-03, -3.1365e-04, -3.3308e-03,  6.5740e-04,\n         3.3836e-03,  1.7968e-03,  5.6052e-45,  9.1714e-04, -3.7225e-11,\n        -1.5972e-03,  3.3398e-03, -4.4637e-04, -3.6734e-19,  1.9321e-06,\n         5.6052e-45, -1.6496e-03,  5.6052e-45, -4.3220e-04, -1.1750e-05,\n        -2.6207e-04,  2.5364e-03,  5.6052e-45,  2.4874e-04, -3.8771e-04,\n         3.1076e-03, -9.5760e-04,  6.0460e-04,  1.1282e-19, -9.6913e-04,\n         1.8747e-03, -2.3665e-03,  5.5657e-03,  3.6068e-04,  8.4913e-05,\n        -3.0548e-05, -1.7480e-03, -1.6406e-03, -8.9434e-04, -1.3305e-04,\n        -3.8913e-03, -5.6052e-45,  2.3141e-03,  1.2096e-03,  1.1739e-03,\n         3.7447e-19,  3.2070e-04, -2.2073e-03,  5.6052e-45,  2.6995e-04,\n         2.6495e-03,  1.7450e-16, -1.6806e-03, -7.4609e-04, -1.5372e-03,\n        -1.7810e-03,  1.6644e-04, -7.2881e-03, -3.3304e-03, -5.3491e-04,\n        -2.6843e-04,  1.1663e-03,  6.3363e-04,  1.8992e-03, -4.1072e-35,\n         5.6052e-45, -7.0372e-03, -7.7995e-04,  6.9158e-05,  3.1582e-04,\n        -3.8041e-03,  2.7068e-03,  5.4373e-04,  5.6052e-45, -1.8535e-03,\n         2.0693e-03, -1.7930e-03, -7.1912e-04,  5.6052e-45, -1.6341e-03,\n        -2.0785e-03, -1.5169e-03, -4.0041e-03,  6.6253e-03, -6.6289e-21,\n         5.6052e-45, -1.3885e-03,  1.3406e-03,  1.6914e-03,  5.4886e-04,\n         1.6931e-04, -4.8455e-04, -3.7299e-04,  5.6052e-45,  3.2643e-04,\n         1.9032e-03,  1.2197e-03,  1.2259e-03,  1.8186e-03,  1.2542e-03,\n         5.6052e-45,  2.3774e-03, -2.8570e-03,  6.9884e-04,  1.1143e-03,\n        -1.3073e-04, -3.2651e-04, -1.5357e-03, -1.7092e-02, -8.1063e-04,\n         3.6122e-03,  7.0732e-04,  3.2542e-03, -4.1671e-04, -2.1162e-04,\n        -8.7467e-05,  3.9853e-03, -1.5770e-03,  1.9945e-03,  5.1694e-04,\n        -2.4642e-05,  1.9110e-03, -8.1750e-03, -4.4143e-03,  3.0264e-03,\n        -5.1558e-05,  3.2541e-03,  2.5530e-03,  5.6052e-45, -4.2672e-04,\n         2.4149e-03, -1.5908e-03,  1.2887e-03, -2.3173e-03,  5.9383e-04,\n         1.4040e-04, -1.5054e-03,  1.8201e-03, -1.0404e-03,  4.2278e-03,\n         1.0953e-03, -8.5257e-04, -2.6585e-04, -3.4092e-04, -1.9889e-03,\n         1.5783e-03,  4.9310e-03, -3.4958e-04, -2.0176e-03, -4.1653e-05,\n         2.1225e-03,  5.6052e-45, -1.2615e-03, -1.5187e-03, -6.9453e-05,\n        -1.2908e-03,  1.5175e-03,  3.8103e-04,  3.2755e-03, -7.3108e-04,\n         3.3495e-03,  2.6484e-03,  5.6052e-45,  1.3955e-04,  2.4664e-03,\n         3.0513e-05,  1.5540e-03,  2.7154e-03,  1.2035e-03, -1.2743e-04,\n         1.5686e-11, -2.2926e-03,  5.6052e-45, -1.1629e-03, -1.0426e-03,\n         4.4875e-03,  3.8890e-04, -1.2673e-03, -2.6926e-03,  5.8198e-04,\n        -7.9894e-04,  5.6052e-45, -1.2130e-03,  1.6454e-03, -7.5367e-03,\n        -1.0624e-05, -3.8889e-03,  1.2439e-03, -2.1511e-03, -1.6852e-03,\n         5.8289e-03, -2.4693e-03, -2.8744e-03,  1.4555e-03,  5.4154e-04,\n        -3.0575e-03,  2.6792e-04,  1.5759e-07, -2.3129e-03, -2.6765e-03,\n        -9.7535e-07, -7.1819e-04,  6.4001e-40,  5.6052e-45, -3.2492e-04,\n        -1.8484e-03,  5.2411e-04,  4.2317e-03, -6.3674e-04, -1.5397e-03,\n         5.6048e-04,  2.3906e-03, -2.8164e-03, -1.1538e-03,  1.4445e-03,\n         3.8265e-03, -9.7848e-04,  4.0255e-04, -8.5947e-04, -8.6640e-05,\n        -4.9727e-05,  1.1320e-03, -2.1908e-05,  3.3552e-03,  5.8964e-06,\n         4.2318e-04, -2.3774e-03,  4.0055e-03, -6.5512e-04,  5.6052e-45,\n         1.2157e-03,  6.3504e-04,  3.2873e-03, -6.5067e-04,  1.6978e-04,\n        -3.8350e-04, -6.3015e-04, -5.1111e-04,  6.1542e-04,  3.4668e-03,\n         8.5541e-03,  5.6052e-45,  7.0538e-04,  1.7820e-03,  1.1006e-04,\n        -6.4889e-04, -3.8285e-04,  4.7411e-04,  4.0403e-03, -1.2390e-04,\n         1.4954e-03,  4.5684e-04, -4.0117e-04, -2.5432e-03,  5.4513e-28,\n         1.0953e-03,  1.2679e-03, -7.2141e-03,  2.2937e-03,  1.0306e-03,\n        -1.4254e-03, -3.4962e-04, -3.8257e-04,  1.2993e-03, -1.0368e-04,\n         1.0880e-03,  2.0327e-03, -7.5284e-04,  7.0328e-04,  1.6928e-03,\n         1.1690e-03,  2.2969e-04, -1.1355e-03,  5.6052e-45,  5.6912e-04,\n         6.9722e-04,  4.9105e-07, -4.4864e-04,  1.5886e-03, -2.5353e-04,\n         1.1741e-03, -5.4882e-03,  4.8552e-04, -2.3815e-03, -4.0432e-04,\n         3.2380e-03,  1.9961e-03,  5.5193e-04, -1.7605e-33,  5.6052e-45,\n         2.6521e-03,  2.4160e-04,  2.9053e-03, -2.0351e-04, -1.7770e-03,\n         2.2448e-03, -1.8172e-03,  2.3603e-03, -9.3427e-04,  4.6545e-04,\n         2.4674e-03, -1.9138e-03,  5.6052e-45,  2.6908e-04,  6.4993e-05,\n         5.6052e-45, -1.0407e-07,  5.6052e-45,  1.9614e-03,  1.4860e-03,\n         1.0679e-04, -7.2474e-04,  5.6052e-45,  8.8939e-04, -2.2507e-03,\n        -2.0403e-04,  2.4861e-04,  1.5738e-04,  4.1969e-03, -9.0950e-04,\n        -4.3497e-07, -4.5330e-03, -4.3736e-16,  2.6728e-03, -3.4864e-12,\n         6.6541e-04, -3.8415e-03, -4.9617e-03,  5.6052e-45, -2.8454e-04,\n         3.2791e-03,  2.8651e-05,  5.0693e-03,  1.8879e-03,  1.4716e-03,\n        -4.9795e-03, -1.4910e-03, -2.4324e-03,  2.8446e-03, -1.5682e-03,\n        -2.8933e-09,  1.4629e-03,  9.7059e-20, -7.9898e-04,  8.4393e-04,\n         5.6052e-45, -1.2870e-03, -1.4444e-03, -3.7080e-04,  2.6747e-03,\n         7.7871e-10, -2.4555e-03, -1.6801e-04,  1.6547e-03,  5.6052e-45,\n         1.5095e-04,  8.6976e-04,  5.6052e-45, -1.3492e-03,  7.0371e-17,\n         1.9926e-03, -1.2434e-05,  5.6052e-45,  4.2391e-03,  1.6444e-03,\n        -2.2978e-04,  1.0726e-03,  2.7664e-03,  6.5306e-04,  3.4669e-03,\n         1.4955e-03,  5.6052e-45, -6.9197e-04,  5.6052e-45,  9.3500e-04,\n         5.7621e-04,  6.1615e-04, -1.7161e-03,  1.6078e-04,  5.0908e-03,\n        -1.7193e-03,  1.2305e-03, -1.9431e-03, -1.9954e-04,  8.6554e-04,\n         6.1657e-44,  3.6795e-03, -2.0908e-03,  4.7156e-04,  3.5966e-04,\n         5.7927e-04,  5.7433e-05, -2.0558e-03,  2.1328e-03, -1.0483e-03,\n         2.0550e-03,  5.6052e-45,  4.8020e-06,  3.4740e-03, -9.1481e-04,\n        -2.3186e-03, -3.0433e-03,  2.2122e-03,  1.7908e-03,  2.8714e-03,\n        -5.2433e-04, -6.5450e-04, -5.8112e-03, -6.1113e-04,  1.2991e-09,\n         5.6052e-45, -3.7407e-04, -8.6636e-04, -7.9024e-04, -1.2939e-03,\n         1.6390e-03, -6.4169e-04, -1.7501e-03,  2.5247e-03,  1.5380e-03,\n         5.4311e-04, -1.9798e-03, -1.3643e-03,  3.7215e-04, -3.3887e-04,\n         1.2315e-12, -6.6939e-04,  4.9933e-03, -3.3018e-03, -5.6052e-45,\n        -9.1870e-03, -2.3953e-03,  9.9825e-34,  5.6052e-45, -1.8366e-03,\n         4.0818e-03, -7.4626e-05,  1.8173e-03,  3.3083e-03, -2.8544e-04,\n        -2.2003e-05,  8.8995e-04, -2.6031e-03, -2.9603e-03,  1.7691e-03,\n         1.1856e-03, -1.0590e-03, -1.7087e-03, -4.7046e-03, -2.6489e-03,\n         9.6220e-04,  1.1795e-03, -6.1423e-03,  2.8327e-03, -1.9739e-04,\n         9.0532e-04, -2.3302e-03, -1.0570e-03,  2.9605e-03,  9.4562e-05,\n        -2.4800e-03, -1.9176e-03,  2.6401e-03, -6.7864e-04,  9.1049e-04,\n         2.5613e-03,  1.8557e-03,  2.7468e-03,  1.7356e-04,  3.0913e-07,\n        -1.6716e-09,  5.6052e-45, -3.9027e-03, -1.2063e-03,  3.5184e-03,\n         4.1062e-03,  4.3985e-22, -1.2497e-03,  4.5070e-04,  1.4553e-03,\n         3.4370e-03,  3.0550e-04,  2.5467e-03, -6.4545e-04, -2.6825e-03,\n         1.6909e-03, -1.3134e-03,  2.0747e-03, -2.3815e-03, -8.7160e-03,\n        -5.2320e-03,  1.6452e-03,  6.0151e-04, -1.9492e-04, -4.6760e-04,\n        -8.1648e-03, -2.6751e-03,  5.5363e-04, -9.6228e-05, -1.5683e-03,\n         4.8903e-04,  2.4498e-03, -2.3832e-03,  1.7399e-03,  1.2952e-03,\n         8.9709e-04, -2.6526e-04, -5.7862e-04,  4.5445e-04,  6.4842e-04,\n        -9.3869e-05, -8.8490e-04,  4.7145e-03,  7.8731e-04,  3.4545e-12,\n         3.1115e-13,  1.7237e-07, -4.2924e-04,  2.8269e-03,  1.4195e-03,\n        -4.3912e-03,  5.2732e-03, -9.6656e-04, -2.4521e-04,  2.0737e-03,\n        -2.2138e-03,  2.4862e-28,  6.3193e-39, -1.6646e-03,  4.4695e-03,\n         1.0923e-03, -1.2242e-03, -7.8823e-04, -5.8591e-22, -2.6610e-04],\n       device='cuda:0')",
-        "exp_avg_sq": "tensor([2.6185e-05, 1.0394e-04, 6.1769e-05, 4.7187e-05, 5.4188e-07, 1.1386e-05,\n        8.2271e-05, 8.5614e-05, 5.1013e-05, 8.3434e-05, 8.1024e-05, 1.4614e-04,\n        1.8464e-06, 7.5689e-05, 8.3315e-05, 8.7148e-05, 1.2984e-04, 1.1138e-04,\n        1.7976e-05, 4.8074e-07, 5.8047e-05, 6.4854e-05, 5.8102e-05, 8.5291e-05,\n        1.7548e-06, 1.3080e-04, 7.9323e-05, 1.1112e-04, 7.1867e-05, 1.0145e-04,\n        8.9637e-05, 1.0222e-04, 3.5326e-06, 3.0992e-07, 6.2278e-05, 8.4035e-05,\n        9.4529e-05, 6.0741e-05, 3.8722e-05, 8.5220e-05, 9.3767e-05, 4.0801e-05,\n        8.1706e-05, 6.9468e-05, 5.5784e-05, 9.4584e-05, 8.3837e-05, 1.4525e-04,\n        1.2973e-04, 3.2615e-05, 1.0640e-04, 1.8409e-07, 2.9872e-07, 2.7956e-07,\n        9.5597e-05, 1.5516e-04, 5.8881e-05, 5.9887e-05, 9.0280e-07, 1.1803e-04,\n        7.7816e-05, 7.0783e-05, 1.2705e-04, 4.5680e-05, 1.0234e-04, 1.2013e-04,\n        8.0419e-05, 3.4217e-07, 1.0126e-04, 8.9997e-05, 3.2717e-06, 1.3977e-06,\n        1.1270e-04, 6.5817e-05, 1.3604e-05, 8.7232e-05, 9.4244e-05, 1.0724e-04,\n        3.5342e-05, 9.8258e-05, 1.2847e-04, 7.9171e-05, 1.0172e-04, 6.4004e-05,\n        8.7273e-05, 6.1845e-05, 9.3616e-05, 3.6464e-05, 8.5587e-05, 7.1024e-05,\n        2.1820e-06, 1.1252e-04, 7.3912e-05, 5.1679e-05, 1.2169e-04, 2.4996e-05,\n        6.7809e-05, 4.7224e-05, 6.6435e-05, 5.3782e-05, 2.7094e-05, 1.7723e-06,\n        1.0651e-04, 1.1359e-04, 1.4858e-04, 2.8774e-05, 7.7910e-05, 1.2017e-04,\n        7.7663e-06, 3.1601e-05, 1.3684e-04, 1.3643e-04, 1.4752e-07, 6.1557e-05,\n        9.7825e-07, 5.3982e-05, 1.0140e-04, 1.3039e-04, 6.5776e-08, 3.2824e-07,\n        2.3927e-07, 6.9650e-05, 1.9053e-07, 1.7587e-04, 7.5585e-05, 1.2484e-04,\n        9.4132e-05, 3.5841e-08, 2.9235e-05, 1.1996e-04, 6.7474e-05, 6.5300e-05,\n        1.0678e-04, 6.3938e-08, 1.0664e-04, 8.1291e-05, 3.6267e-05, 8.1214e-05,\n        5.7613e-05, 8.0973e-05, 1.4920e-04, 9.8473e-05, 1.0549e-04, 8.5102e-05,\n        9.9494e-05, 8.1772e-05, 2.8215e-06, 9.8957e-05, 7.6632e-05, 6.0016e-05,\n        2.4200e-06, 5.4160e-05, 6.3808e-05, 1.2984e-06, 7.0577e-05, 1.0513e-04,\n        2.5717e-06, 1.1417e-04, 6.7006e-05, 9.2386e-05, 6.7913e-06, 8.7186e-05,\n        1.9947e-04, 7.3766e-05, 7.8754e-05, 6.3501e-05, 1.4718e-04, 5.0104e-05,\n        7.4427e-05, 3.0491e-07, 4.7086e-07, 9.1470e-05, 1.1657e-04, 6.9807e-05,\n        6.6640e-05, 1.1796e-04, 3.5256e-05, 9.9528e-05, 8.5455e-08, 3.3643e-05,\n        7.4284e-05, 1.1369e-04, 6.7078e-06, 6.2876e-07, 1.0280e-04, 9.3019e-05,\n        1.2655e-04, 4.8381e-05, 6.5508e-05, 8.3013e-08, 2.0920e-06, 8.1143e-05,\n        4.1950e-05, 8.5873e-05, 9.3965e-05, 7.8254e-05, 5.4772e-05, 1.0383e-04,\n        3.2973e-07, 7.5674e-05, 5.0118e-05, 4.2554e-05, 1.3158e-04, 3.2573e-05,\n        1.7879e-04, 1.6264e-08, 8.9628e-05, 9.3593e-05, 7.0992e-05, 1.0373e-04,\n        4.2838e-05, 8.6818e-05, 9.0367e-05, 8.9597e-05, 4.0722e-05, 7.4222e-05,\n        5.1326e-05, 1.0765e-04, 1.0529e-04, 4.9929e-05, 6.4792e-05, 9.1554e-05,\n        6.7870e-05, 7.0876e-05, 7.7211e-05, 7.0628e-05, 1.3262e-04, 1.4059e-04,\n        5.7287e-05, 1.2511e-04, 8.2398e-05, 1.4213e-04, 1.0192e-04, 4.0426e-07,\n        9.4356e-05, 4.8902e-05, 7.8567e-05, 6.1416e-05, 1.8999e-04, 7.6075e-05,\n        4.4971e-05, 6.0220e-05, 8.8060e-05, 8.6001e-05, 1.3341e-04, 5.4736e-05,\n        3.7458e-05, 8.9258e-05, 5.9941e-05, 8.1535e-05, 7.6555e-05, 7.9841e-05,\n        6.3156e-05, 7.4600e-05, 1.0078e-04, 8.0634e-05, 1.7419e-06, 6.4350e-05,\n        4.5434e-05, 1.1492e-05, 8.4327e-05, 1.2536e-04, 9.3495e-05, 9.0823e-05,\n        8.5464e-05, 9.2246e-05, 6.1173e-05, 2.5820e-07, 8.8737e-05, 9.7515e-05,\n        6.1668e-05, 1.1619e-04, 7.8469e-05, 2.3436e-05, 1.1286e-04, 4.5058e-07,\n        8.2334e-05, 1.2733e-06, 5.7528e-05, 1.5404e-05, 7.8400e-05, 6.4173e-05,\n        1.3407e-04, 1.0631e-04, 1.0481e-04, 8.8875e-05, 5.1930e-08, 9.9428e-05,\n        1.1994e-04, 5.9341e-05, 1.1065e-06, 8.9699e-05, 1.1248e-04, 9.9404e-05,\n        7.6766e-05, 1.1760e-04, 1.3545e-04, 7.7120e-05, 8.2723e-05, 1.5759e-04,\n        6.8220e-05, 5.4418e-06, 3.6677e-06, 6.0936e-05, 8.8757e-05, 1.0897e-05,\n        9.4508e-05, 5.3596e-08, 1.0559e-06, 8.7500e-05, 1.1486e-04, 5.9288e-05,\n        9.1286e-05, 1.0201e-04, 7.0026e-05, 9.0122e-05, 1.0238e-04, 8.5591e-05,\n        5.7840e-05, 9.6669e-05, 1.0544e-04, 1.4472e-04, 1.2356e-04, 5.6766e-05,\n        2.4556e-05, 9.6125e-05, 7.5443e-05, 9.2146e-07, 6.3154e-05, 1.0093e-04,\n        1.0514e-05, 5.2251e-05, 1.1961e-04, 9.2645e-05, 1.2597e-10, 7.3657e-05,\n        4.6807e-05, 1.2449e-04, 5.7429e-05, 7.0788e-05, 1.1652e-05, 9.9527e-05,\n        7.5082e-05, 7.2245e-05, 7.6273e-05, 1.3233e-04, 1.8434e-06, 4.5481e-06,\n        8.8133e-05, 9.6553e-05, 5.2322e-05, 1.0399e-04, 7.0010e-05, 8.0560e-05,\n        1.3993e-04, 6.5921e-05, 7.4668e-05, 7.2204e-05, 1.0716e-04, 1.7566e-06,\n        4.2006e-05, 5.0364e-05, 9.5045e-05, 1.1070e-04, 9.8440e-05, 2.0420e-04,\n        1.8978e-05, 9.6445e-05, 1.0712e-04, 1.0500e-04, 1.0441e-04, 8.0089e-05,\n        7.3783e-05, 4.5673e-06, 7.9015e-05, 1.3546e-04, 1.8708e-04, 1.9370e-05,\n        8.0249e-07, 1.0890e-04, 4.9397e-06, 4.3785e-06, 7.1149e-05, 1.5961e-04,\n        5.1601e-05, 6.7308e-05, 8.8343e-05, 1.2766e-04, 5.8176e-05, 4.6119e-05,\n        1.1389e-04, 1.0290e-04, 6.7217e-05, 4.3518e-07, 1.4440e-06, 9.5064e-05,\n        9.4383e-05, 9.7615e-05, 1.1845e-04, 5.2660e-05, 6.8345e-05, 6.1756e-05,\n        1.4186e-05, 5.7426e-05, 6.6783e-05, 1.2038e-04, 6.5310e-05, 5.3178e-07,\n        1.4157e-06, 2.1271e-05, 1.5319e-07, 1.8450e-06, 2.6727e-07, 1.1933e-04,\n        8.5703e-05, 2.4135e-05, 5.8756e-05, 7.2158e-07, 3.5566e-05, 7.8141e-05,\n        5.5834e-05, 9.3492e-05, 1.0075e-04, 1.3223e-04, 7.8316e-05, 1.8754e-06,\n        4.3172e-05, 5.4594e-07, 4.6574e-05, 1.0072e-07, 3.9667e-05, 9.7900e-05,\n        8.4505e-05, 4.6350e-07, 5.1821e-05, 7.8551e-05, 1.1745e-04, 1.1118e-04,\n        6.1050e-05, 3.2160e-05, 9.7376e-05, 4.0605e-05, 9.7365e-05, 1.2358e-04,\n        5.8186e-05, 1.2478e-06, 7.2198e-05, 1.5644e-07, 7.9856e-05, 4.7343e-05,\n        5.1903e-06, 2.6853e-05, 6.3829e-05, 5.3001e-06, 8.1505e-05, 2.4521e-07,\n        4.6788e-05, 5.6387e-06, 7.1344e-05, 7.0069e-06, 8.7032e-05, 1.2208e-04,\n        4.5556e-08, 7.1572e-05, 5.8981e-06, 9.8941e-05, 6.0820e-05, 3.7207e-07,\n        9.8656e-05, 5.5876e-05, 1.6058e-04, 1.2404e-04, 6.0918e-05, 1.4882e-05,\n        5.3910e-05, 8.8351e-05, 2.5861e-06, 1.0310e-04, 8.2110e-07, 6.7687e-06,\n        3.4286e-06, 3.0858e-06, 6.5671e-05, 9.9121e-05, 1.5407e-04, 8.6984e-05,\n        7.1729e-05, 6.9371e-05, 8.7477e-05, 6.8605e-05, 3.4451e-07, 8.3578e-05,\n        8.1260e-05, 1.5208e-04, 4.7259e-06, 2.0069e-05, 6.8325e-05, 5.7288e-05,\n        7.2662e-05, 6.0905e-06, 7.6995e-05, 2.5564e-06, 6.0025e-07, 1.2225e-04,\n        7.8866e-05, 1.3945e-04, 8.2698e-05, 7.7207e-05, 9.0479e-05, 6.0077e-05,\n        9.0126e-05, 1.6558e-04, 6.1826e-05, 8.6941e-05, 1.9133e-06, 2.1177e-06,\n        1.0096e-04, 1.9103e-04, 1.1656e-04, 7.5653e-05, 9.0906e-05, 9.2470e-05,\n        8.5884e-05, 8.2854e-05, 7.5234e-05, 1.4954e-05, 8.3693e-05, 9.7825e-05,\n        6.3007e-05, 5.4483e-05, 2.5304e-07, 5.8177e-05, 9.6390e-05, 5.2218e-05,\n        1.9278e-06, 7.9757e-05, 8.1680e-05, 1.5617e-07, 4.6986e-07, 9.3479e-05,\n        9.5239e-05, 1.0603e-04, 5.2933e-05, 1.0776e-04, 5.5441e-05, 9.0652e-05,\n        6.6375e-05, 4.8456e-05, 5.6797e-05, 8.7112e-05, 9.3751e-05, 9.1886e-05,\n        1.0527e-04, 1.0298e-04, 9.0333e-05, 4.2948e-05, 7.7361e-05, 2.2191e-04,\n        6.5279e-05, 7.3744e-05, 3.9408e-05, 7.9760e-05, 9.1075e-05, 1.4052e-04,\n        1.0290e-06, 6.9951e-05, 1.1078e-04, 4.7856e-05, 1.0981e-04, 8.3493e-05,\n        8.2707e-05, 8.2696e-05, 8.3315e-05, 1.2272e-04, 4.3915e-07, 2.4494e-07,\n        2.8034e-06, 8.8715e-05, 7.5586e-05, 1.0030e-04, 9.2822e-05, 5.3358e-06,\n        6.6156e-05, 8.0489e-05, 2.5207e-05, 7.7702e-05, 5.6717e-05, 6.4486e-05,\n        1.1964e-04, 9.9142e-05, 6.0166e-05, 1.1964e-04, 9.7669e-05, 1.2682e-04,\n        9.5410e-05, 7.4313e-05, 9.2821e-05, 9.2618e-05, 1.3747e-04, 1.4180e-04,\n        1.2441e-04, 5.9513e-05, 7.8540e-05, 7.1513e-05, 8.1343e-05, 6.1158e-05,\n        6.6370e-05, 8.8826e-05, 1.7511e-04, 1.3670e-04, 1.1294e-04, 1.2170e-04,\n        9.5316e-05, 2.8118e-05, 3.2750e-05, 9.4559e-05, 8.2394e-05, 8.8387e-05,\n        4.8002e-05, 9.3894e-08, 3.5055e-06, 2.6614e-07, 1.0201e-04, 7.1645e-05,\n        8.9279e-05, 1.1596e-04, 7.1520e-05, 5.8988e-05, 5.3306e-05, 1.1040e-04,\n        5.0714e-05, 8.9861e-07, 4.1866e-09, 1.1326e-04, 7.4199e-05, 1.3079e-04,\n        1.0399e-04, 8.0540e-05, 3.3352e-07, 6.1059e-05], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([ 5.1233e-04,  1.4962e-03,  8.6219e-04,  6.6010e-04, -2.8160e-12,\n         9.4107e-05,  1.1703e-04,  5.3116e-04,  1.1135e-03,  2.2131e-03,\n        -7.7125e-04,  3.4627e-03,  5.6052e-45, -6.5682e-04,  1.2157e-03,\n        -3.2750e-04, -2.4690e-03,  2.3180e-03,  5.7181e-04,  5.6052e-45,\n        -2.4293e-03, -2.9330e-03, -1.1394e-03, -1.1585e-03,  5.6052e-45,\n         2.7927e-03,  1.9219e-04, -7.7537e-04,  1.0711e-03, -2.8580e-03,\n        -8.0813e-04, -2.4576e-04, -2.3504e-08,  5.6052e-45, -1.2613e-03,\n         7.3082e-04,  2.2089e-04, -2.8135e-03, -1.7028e-04, -9.0479e-04,\n        -3.4984e-03, -2.0797e-03,  3.1206e-04, -1.5827e-03,  1.4216e-03,\n         5.5516e-03,  3.3014e-03, -6.2577e-04,  1.7331e-03,  1.2679e-03,\n        -4.2741e-05,  5.6052e-45, -5.6052e-45,  5.6052e-45, -1.2496e-03,\n        -9.4733e-04,  1.6383e-03, -4.1002e-04,  3.2997e-09,  3.3793e-03,\n        -3.7731e-04,  3.8605e-04, -2.8461e-03, -1.1535e-03, -1.6822e-03,\n        -1.6403e-03,  7.6820e-05,  5.6052e-45,  1.1597e-04,  2.3749e-03,\n        -2.1615e-04,  5.6052e-45,  3.5370e-03,  3.2385e-03,  2.5593e-04,\n         1.1675e-03, -1.5430e-03, -1.2461e-03,  3.1295e-03, -9.0900e-03,\n         2.0511e-03, -1.5631e-03,  1.7974e-03, -5.7405e-04,  1.3671e-03,\n        -8.7397e-04,  2.9302e-03,  1.2321e-03,  1.2652e-03,  2.4025e-03,\n         5.6052e-45, -1.0113e-03, -8.0032e-04, -3.4159e-03, -9.0481e-04,\n        -1.8566e-03,  7.8791e-04,  5.2959e-04, -3.0380e-03, -6.7021e-04,\n         7.4207e-04,  6.3742e-04, -1.4810e-03, -1.3313e-03, -4.9065e-04,\n        -1.0980e-04,  4.6380e-05,  2.6412e-03, -1.7604e-04, -7.9487e-04,\n         4.2142e-04, -1.7397e-03,  5.6052e-45,  1.0810e-03, -1.6093e-05,\n         1.8620e-03, -1.1098e-03,  4.2065e-03, -5.6052e-45,  1.0830e-05,\n         5.6052e-45, -4.9362e-04,  5.6052e-45,  4.5340e-03,  2.7321e-04,\n         3.2198e-03, -6.2468e-03,  5.6052e-45,  4.0794e-04, -2.9376e-04,\n         2.0729e-03,  1.9686e-03,  2.8169e-03,  9.9771e-25,  1.8688e-03,\n        -3.2034e-04,  2.9599e-04,  1.7660e-03,  5.4940e-04, -5.5974e-04,\n         1.7476e-03,  2.0100e-03,  5.2797e-03,  2.3488e-03,  5.5077e-03,\n         2.2498e-03, -8.5076e-13,  4.0588e-03, -4.4685e-04,  2.2580e-03,\n        -2.3057e-17,  1.3667e-03, -2.0953e-03,  5.6052e-45,  3.2642e-03,\n         1.1543e-03,  6.5475e-20,  2.3212e-03,  3.4358e-03, -4.1328e-04,\n        -1.2298e-03,  4.9247e-04,  6.0939e-04,  3.2708e-03,  2.0755e-04,\n         3.8615e-05,  9.2578e-04,  1.8436e-04,  2.7992e-03,  7.7843e-23,\n         5.6052e-45, -6.7832e-04, -2.2660e-03,  8.3994e-04, -2.6474e-03,\n         5.8393e-04, -7.7083e-04, -2.7944e-03,  5.6052e-45,  1.2081e-03,\n        -1.4105e-03, -1.8192e-03,  1.4575e-04,  5.6052e-45, -5.0196e-04,\n         1.1285e-03, -8.7956e-04, -7.5027e-04, -1.9143e-05,  5.6052e-45,\n         5.6052e-45,  2.1472e-03,  2.0672e-03, -4.0834e-03,  1.1510e-04,\n         7.5289e-04, -1.6370e-03,  1.5778e-03,  5.6052e-45,  9.0714e-04,\n         1.8977e-03, -1.8082e-03,  9.2614e-04, -1.9589e-03, -1.1776e-03,\n         5.6052e-45,  3.3800e-04,  3.8359e-04,  1.0382e-03,  1.8186e-03,\n         7.9169e-04,  5.4020e-04, -4.6689e-04,  3.7264e-04,  5.1813e-04,\n        -2.6652e-03,  1.9006e-03, -1.2771e-03, -2.5275e-03, -3.0154e-03,\n         5.3493e-04,  2.4243e-04,  1.5057e-03, -1.1234e-03,  2.1051e-03,\n         9.7383e-04, -3.3279e-03, -3.1169e-04,  2.4938e-04, -2.2146e-06,\n         7.9057e-04, -2.7659e-04, -1.0425e-03,  5.6052e-45, -1.8182e-03,\n         3.0576e-03,  2.8684e-03,  4.9024e-04,  1.5073e-03,  2.0503e-03,\n         7.2662e-04,  5.9012e-04, -1.2612e-03,  8.4114e-04,  1.4172e-04,\n         3.8061e-03, -1.0976e-03, -4.4220e-04, -1.6167e-03,  2.3890e-03,\n        -3.2845e-03,  3.0843e-03,  1.2070e-04,  1.6564e-03,  5.6408e-04,\n        -1.9539e-03,  5.6052e-45, -2.9932e-04, -2.6776e-04,  5.9844e-04,\n        -1.4972e-04,  7.2556e-04, -1.3185e-03,  8.6790e-04, -1.6135e-03,\n        -5.3571e-04, -2.9040e-04,  5.6052e-45, -2.0915e-03, -2.2987e-03,\n         5.6645e-04, -8.4885e-04,  9.0699e-04,  3.9753e-04,  1.3428e-04,\n         5.6052e-45,  1.8292e-03,  5.6052e-45, -4.4438e-03,  8.0182e-04,\n        -1.1643e-03,  4.3945e-03, -2.2254e-03, -5.2929e-03, -3.7621e-04,\n         3.2349e-04, -4.3990e-19,  3.2879e-03,  5.5551e-04, -2.1160e-05,\n        -1.6796e-04,  2.1284e-03,  6.7839e-04,  1.9942e-03, -1.7185e-04,\n         2.9146e-03, -5.7036e-03,  4.0484e-03,  3.0776e-03, -6.2567e-04,\n         1.2639e-03,  3.4377e-04, -4.9843e-06,  1.6383e-04,  4.6551e-04,\n         2.1297e-05, -2.2879e-04,  5.6052e-45,  5.6052e-45,  5.6641e-04,\n         1.3571e-03, -3.0112e-03, -5.2022e-04,  1.7431e-03, -8.2426e-04,\n        -4.6999e-04,  1.4273e-03,  1.8569e-03, -3.6938e-04,  1.7993e-04,\n         7.7306e-04, -3.0861e-04, -2.5285e-03, -8.6260e-05,  2.4698e-04,\n        -1.5429e-03,  1.3110e-03, -5.4266e-04,  1.9959e-05,  1.3702e-03,\n         2.2933e-04,  1.4004e-03, -1.4422e-03,  1.7862e-03,  5.6052e-45,\n        -2.2920e-04,  1.7263e-04, -9.7651e-04,  1.5978e-03, -3.9989e-04,\n        -6.9877e-04,  1.0665e-03,  3.1147e-03,  9.5281e-04, -1.3571e-03,\n        -2.0422e-03,  5.6052e-45, -3.1476e-04, -1.0402e-03,  1.6093e-03,\n         5.0974e-04, -1.3595e-03,  1.8338e-03,  1.0209e-03, -1.0805e-03,\n        -6.0864e-03, -4.5676e-04,  2.0731e-03,  2.0280e-03, -1.4483e-03,\n        -1.5112e-03, -2.0867e-04,  3.4876e-03,  5.2873e-04, -4.3349e-03,\n         1.6431e-03,  3.3320e-04, -5.3557e-04, -1.1041e-04,  1.0357e-03,\n         3.0968e-03,  7.0172e-04,  2.3487e-04, -1.8611e-03, -1.2893e-03,\n         2.4788e-03, -3.5122e-04,  7.0537e-05,  5.6052e-45, -1.3271e-03,\n        -2.2722e-03,  5.6052e-45, -7.8819e-04, -3.5749e-03,  1.7267e-03,\n        -1.9830e-03, -1.6744e-03,  2.2860e-03,  1.9861e-04,  2.0112e-04,\n         3.5650e-04, -2.7359e-03,  1.4410e-03, -5.6052e-45,  5.6052e-45,\n        -8.9577e-04,  3.5788e-03,  1.2752e-03,  3.6856e-03, -3.6852e-04,\n         2.5619e-04,  2.8084e-04, -1.5861e-03, -4.6875e-04, -5.3986e-04,\n         9.9002e-04,  2.3711e-04,  5.6052e-45, -3.6677e-03, -9.2103e-04,\n         5.6052e-45, -8.7159e-04,  5.6052e-45, -4.0697e-05, -8.5340e-04,\n         3.2206e-04,  7.8644e-04,  5.6052e-45, -1.5146e-03,  3.0435e-03,\n        -6.3783e-04, -1.3443e-03, -1.6531e-03, -2.1750e-03,  1.1121e-03,\n         7.2222e-04, -2.5241e-03,  6.5804e-24, -2.7617e-03,  2.8942e-12,\n        -6.0669e-04,  2.4579e-04,  6.0150e-04,  5.6052e-45, -6.9064e-05,\n         1.4827e-03, -4.3664e-04,  4.7341e-04, -1.2992e-03, -3.8477e-04,\n         3.5847e-03, -6.8151e-04,  1.3217e-03, -1.0765e-03, -4.6359e-04,\n         7.9395e-04, -2.2083e-03,  5.6052e-45, -2.3358e-03,  4.1295e-03,\n         5.6052e-45,  2.1284e-03,  2.2264e-04,  6.1589e-04, -6.9541e-04,\n         3.0862e-15, -6.7634e-04,  5.3916e-06,  5.7845e-04,  5.6052e-45,\n        -8.4421e-04, -6.4071e-03,  5.6052e-45, -6.1570e-04,  1.1874e-10,\n         1.6721e-03,  1.0634e-03,  5.6052e-45,  5.2167e-04, -4.3601e-04,\n         5.2242e-03,  7.0871e-04, -5.1759e-03, -1.8040e-03, -2.1591e-03,\n        -5.3001e-05,  5.6052e-45,  3.0465e-03,  5.6052e-45, -2.9522e-04,\n        -1.0857e-04, -5.6573e-04, -1.4785e-03,  2.4057e-03,  5.6347e-04,\n        -5.0183e-04,  2.1063e-03, -2.7572e-04, -3.3027e-03,  1.9016e-03,\n         5.6052e-45, -9.0800e-04,  7.3842e-04,  1.9143e-03, -5.9897e-04,\n         1.0573e-04,  3.2216e-03, -1.4969e-04, -9.8013e-05, -8.7615e-04,\n         2.4830e-03,  5.6052e-45, -3.5623e-04,  2.7601e-03, -1.5760e-04,\n        -6.6513e-04, -4.0614e-06,  6.0204e-04, -2.1472e-03,  2.3123e-03,\n         2.0108e-03, -3.6236e-03, -1.8755e-03,  4.0671e-04, -3.7499e-13,\n         5.6052e-45, -2.5051e-04, -2.9165e-03, -1.5708e-03,  6.8843e-04,\n         1.6382e-03,  2.6966e-03,  2.3469e-03,  8.3923e-04, -6.5503e-04,\n        -5.1303e-04, -1.9877e-03, -8.5651e-04, -9.4771e-04, -3.1107e-03,\n         2.7734e-22,  1.8081e-03,  1.3143e-03,  4.3692e-04, -5.6052e-45,\n         6.7914e-04,  1.3574e-05,  5.6052e-45,  5.6052e-45,  1.9418e-03,\n         4.6115e-04,  1.3270e-03,  1.6926e-03, -1.7398e-03, -5.8611e-05,\n        -1.4337e-03,  1.4241e-03,  6.2230e-04,  2.5681e-03,  7.9564e-04,\n        -7.1825e-04,  2.9431e-04,  8.9161e-04, -5.4874e-04, -1.8233e-03,\n        -3.6833e-04,  8.9179e-04, -8.6929e-04, -9.3005e-04,  3.4687e-03,\n         5.7667e-04,  2.3803e-03, -4.0191e-04,  3.5921e-03,  6.4549e-04,\n        -2.6844e-03, -3.5730e-03, -4.3390e-04,  3.9128e-04, -9.8408e-04,\n         4.1179e-03,  5.0758e-04, -1.0377e-03,  1.0575e-03, -2.8883e-04,\n        -3.8610e-35,  5.6052e-45, -2.5155e-03, -1.5328e-03,  3.0458e-03,\n        -8.3949e-04,  5.6052e-45,  1.3146e-03, -5.2305e-03, -1.0560e-03,\n        -8.0811e-04, -3.0424e-03,  4.5300e-04, -2.0464e-03, -2.4431e-03,\n         4.9951e-04,  6.8201e-04, -3.1527e-04, -8.7993e-04, -2.9484e-03,\n         4.8691e-04, -1.7727e-03, -7.9813e-04, -2.3738e-03, -2.6145e-05,\n        -1.8552e-03, -1.0963e-03,  1.2725e-04, -2.3126e-03, -6.4813e-04,\n         1.9604e-04, -2.7641e-03,  9.3641e-04,  9.5629e-04,  2.4258e-03,\n        -1.0593e-03,  1.5176e-03, -2.2232e-04,  1.4156e-03,  3.2536e-04,\n         4.4908e-03, -1.0206e-03,  9.0033e-08,  1.5631e-03,  1.4958e-10,\n         2.7674e-14, -1.5015e-05,  2.3728e-03,  1.6508e-04, -3.3530e-03,\n         2.2802e-03,  7.1558e-04, -6.1278e-04,  1.3469e-03, -4.3877e-04,\n        -6.6916e-03, -3.7049e-34,  8.0133e-06,  1.9363e-03,  2.4436e-04,\n        -2.1892e-03, -1.7682e-03, -1.6809e-03,  4.8598e-31,  3.1390e-03],\n       device='cuda:0')",
+        "exp_avg_sq": "tensor([1.8429e-05, 7.8424e-05, 4.1216e-05, 3.2207e-05, 1.5486e-07, 8.1643e-06,\n        5.6661e-05, 5.4997e-05, 3.6809e-05, 5.8048e-05, 5.5900e-05, 1.0907e-04,\n        5.2763e-07, 4.9280e-05, 6.0358e-05, 6.0430e-05, 9.3022e-05, 6.6292e-05,\n        2.2263e-05, 1.3737e-07, 4.6491e-05, 4.1435e-05, 3.8910e-05, 6.2517e-05,\n        5.0144e-07, 8.2325e-05, 5.6426e-05, 7.6278e-05, 5.5761e-05, 6.7651e-05,\n        6.4038e-05, 6.5705e-05, 1.0096e-06, 8.8563e-08, 4.7941e-05, 5.5138e-05,\n        6.3255e-05, 4.0000e-05, 3.1811e-05, 5.8204e-05, 6.2877e-05, 3.0693e-05,\n        5.5385e-05, 4.5257e-05, 4.4873e-05, 6.5875e-05, 5.8716e-05, 9.9101e-05,\n        9.7689e-05, 3.3143e-05, 7.1325e-05, 5.2607e-08, 8.5362e-08, 7.9888e-08,\n        7.1962e-05, 1.1149e-04, 4.4462e-05, 4.1195e-05, 2.5838e-07, 6.9597e-05,\n        5.9119e-05, 4.8329e-05, 9.5072e-05, 3.5547e-05, 6.7055e-05, 7.1558e-05,\n        5.2508e-05, 9.7777e-08, 7.3431e-05, 6.2996e-05, 7.9860e-06, 3.9941e-07,\n        7.2067e-05, 4.6330e-05, 1.3939e-05, 6.3709e-05, 7.0940e-05, 7.1251e-05,\n        3.4550e-05, 7.3103e-05, 9.4840e-05, 5.4518e-05, 7.9089e-05, 4.5982e-05,\n        6.0059e-05, 4.9573e-05, 5.8442e-05, 3.0440e-05, 6.0963e-05, 5.4131e-05,\n        6.2351e-07, 7.2436e-05, 5.1906e-05, 4.2414e-05, 9.1787e-05, 2.1514e-05,\n        5.2725e-05, 4.1873e-05, 4.9157e-05, 3.6346e-05, 2.9048e-05, 5.7960e-06,\n        7.6109e-05, 8.0546e-05, 1.2031e-04, 3.1543e-05, 5.6253e-05, 8.2481e-05,\n        1.2189e-05, 2.6002e-05, 8.6256e-05, 9.0572e-05, 4.2156e-08, 4.1130e-05,\n        5.3011e-07, 4.5521e-05, 6.3898e-05, 9.7266e-05, 1.8796e-08, 9.8546e-08,\n        6.8374e-08, 5.0582e-05, 5.4445e-08, 1.1226e-04, 5.4902e-05, 7.8698e-05,\n        7.3919e-05, 1.0242e-08, 2.3582e-05, 7.7462e-05, 4.4632e-05, 5.0593e-05,\n        7.1332e-05, 1.8271e-08, 6.7731e-05, 5.8945e-05, 3.1110e-05, 5.1993e-05,\n        4.2545e-05, 5.2321e-05, 1.0764e-04, 6.3356e-05, 7.3797e-05, 5.5977e-05,\n        6.8316e-05, 6.1020e-05, 8.0626e-07, 6.7718e-05, 5.7531e-05, 4.4023e-05,\n        6.9153e-07, 3.8952e-05, 4.7206e-05, 3.7102e-07, 4.9116e-05, 7.3234e-05,\n        7.3487e-07, 7.9238e-05, 4.3922e-05, 6.1611e-05, 2.1392e-05, 5.9273e-05,\n        1.4000e-04, 5.4958e-05, 5.5462e-05, 4.1322e-05, 1.0242e-04, 3.5023e-05,\n        5.3783e-05, 8.7312e-08, 1.3455e-07, 6.2080e-05, 8.8259e-05, 4.6051e-05,\n        5.0702e-05, 8.2736e-05, 2.5034e-05, 6.8924e-05, 2.4419e-08, 2.8527e-05,\n        5.2167e-05, 7.3348e-05, 9.0304e-06, 1.7967e-07, 6.7260e-05, 6.1738e-05,\n        8.1591e-05, 3.5377e-05, 4.5612e-05, 5.2171e-08, 5.9779e-07, 5.3762e-05,\n        2.8652e-05, 7.1083e-05, 6.1304e-05, 5.5979e-05, 3.9565e-05, 7.5861e-05,\n        9.4224e-08, 4.8972e-05, 4.0873e-05, 2.9134e-05, 8.5371e-05, 2.5459e-05,\n        1.1806e-04, 4.6474e-09, 5.6294e-05, 6.2230e-05, 4.8022e-05, 6.4653e-05,\n        3.1432e-05, 5.7633e-05, 6.0467e-05, 5.9572e-05, 3.4891e-05, 5.2127e-05,\n        3.9065e-05, 7.4697e-05, 7.3555e-05, 3.6516e-05, 4.0428e-05, 6.5586e-05,\n        4.5840e-05, 5.1340e-05, 5.2006e-05, 5.3998e-05, 8.8808e-05, 9.3350e-05,\n        3.8006e-05, 8.3215e-05, 6.1270e-05, 1.0445e-04, 7.0716e-05, 1.1552e-07,\n        6.6224e-05, 3.3235e-05, 5.2076e-05, 3.9190e-05, 1.3412e-04, 5.7846e-05,\n        3.7738e-05, 5.5868e-05, 5.8193e-05, 5.1935e-05, 9.0579e-05, 3.8098e-05,\n        3.0676e-05, 7.1407e-05, 4.0514e-05, 5.2177e-05, 5.7026e-05, 5.2525e-05,\n        4.3748e-05, 4.9606e-05, 6.8073e-05, 5.4826e-05, 4.9775e-07, 4.4180e-05,\n        3.3553e-05, 1.3345e-05, 5.5598e-05, 8.0616e-05, 6.8480e-05, 6.4118e-05,\n        5.9491e-05, 5.7352e-05, 4.2876e-05, 7.3783e-08, 6.4196e-05, 6.4906e-05,\n        4.3185e-05, 8.3245e-05, 4.9406e-05, 2.4759e-05, 7.4066e-05, 1.2876e-07,\n        5.5531e-05, 3.6385e-07, 4.0812e-05, 1.8832e-05, 5.4092e-05, 4.6169e-05,\n        8.6733e-05, 7.8781e-05, 6.8829e-05, 6.6240e-05, 1.4839e-08, 7.6035e-05,\n        7.8169e-05, 4.0586e-05, 6.9116e-06, 5.9455e-05, 8.6081e-05, 6.8788e-05,\n        4.7999e-05, 9.0710e-05, 9.8244e-05, 5.5921e-05, 5.4694e-05, 1.1449e-04,\n        4.9329e-05, 6.2151e-06, 1.1886e-06, 4.3558e-05, 5.7345e-05, 3.4039e-06,\n        6.2881e-05, 1.5315e-08, 3.0172e-07, 5.8508e-05, 8.4363e-05, 3.9821e-05,\n        6.2632e-05, 7.2649e-05, 5.8004e-05, 5.8830e-05, 6.8701e-05, 5.1764e-05,\n        4.0504e-05, 6.9099e-05, 7.2059e-05, 1.0150e-04, 8.1962e-05, 4.0281e-05,\n        1.9336e-05, 6.2384e-05, 4.9710e-05, 7.0670e-06, 4.2761e-05, 6.5936e-05,\n        1.1104e-05, 3.6789e-05, 7.6847e-05, 7.2629e-05, 3.5995e-11, 5.1730e-05,\n        3.6673e-05, 8.5187e-05, 3.6331e-05, 4.6995e-05, 9.1364e-06, 6.6941e-05,\n        4.9654e-05, 5.2543e-05, 5.2972e-05, 8.5916e-05, 5.2677e-07, 6.7806e-06,\n        5.9184e-05, 5.9994e-05, 4.2475e-05, 6.7451e-05, 4.9160e-05, 5.1374e-05,\n        9.5346e-05, 5.5690e-05, 5.1087e-05, 5.1013e-05, 7.7113e-05, 1.3310e-06,\n        2.9492e-05, 3.7743e-05, 6.4588e-05, 8.8312e-05, 6.5177e-05, 1.4898e-04,\n        1.7569e-05, 6.9014e-05, 7.9805e-05, 6.7745e-05, 6.6396e-05, 4.6284e-05,\n        4.9685e-05, 1.0765e-05, 5.2503e-05, 9.4992e-05, 1.2216e-04, 1.7051e-05,\n        2.2932e-07, 6.6446e-05, 1.9666e-05, 1.2512e-06, 5.0863e-05, 1.0809e-04,\n        4.8287e-05, 4.1113e-05, 5.7037e-05, 7.6571e-05, 4.0140e-05, 3.2797e-05,\n        7.5846e-05, 7.5232e-05, 4.8110e-05, 1.2436e-07, 4.1264e-07, 5.8930e-05,\n        6.4290e-05, 5.9733e-05, 8.5655e-05, 4.2477e-05, 5.1533e-05, 4.0953e-05,\n        1.9062e-05, 4.4839e-05, 4.8009e-05, 8.5967e-05, 4.2455e-05, 1.5196e-07,\n        8.6692e-06, 1.7067e-05, 4.3776e-08, 5.9799e-06, 7.6373e-08, 7.7939e-05,\n        6.6996e-05, 2.2114e-05, 4.4705e-05, 2.0620e-07, 2.7376e-05, 6.1274e-05,\n        3.9801e-05, 6.7514e-05, 7.7499e-05, 8.3942e-05, 5.4821e-05, 3.9531e-06,\n        3.7451e-05, 1.5607e-07, 3.2283e-05, 2.8839e-08, 3.2195e-05, 6.9658e-05,\n        5.9980e-05, 1.3245e-07, 3.5261e-05, 5.4220e-05, 7.9021e-05, 6.9342e-05,\n        4.8851e-05, 3.0698e-05, 7.3776e-05, 3.0239e-05, 6.1269e-05, 8.3531e-05,\n        4.3949e-05, 1.2398e-06, 5.4793e-05, 4.4704e-08, 5.6602e-05, 3.6121e-05,\n        1.4832e-06, 2.8724e-05, 4.3440e-05, 4.2151e-06, 5.2914e-05, 7.0113e-08,\n        3.5470e-05, 5.1513e-06, 4.5728e-05, 2.0023e-06, 5.5838e-05, 9.5254e-05,\n        1.3018e-08, 4.7710e-05, 1.6855e-06, 6.7336e-05, 4.4844e-05, 1.0632e-07,\n        6.9499e-05, 3.8512e-05, 1.1443e-04, 7.7913e-05, 4.8103e-05, 1.9658e-05,\n        3.8010e-05, 5.8789e-05, 7.3901e-07, 6.7558e-05, 2.3464e-07, 9.9241e-06,\n        6.0042e-06, 7.3457e-06, 4.9822e-05, 6.4646e-05, 1.1142e-04, 5.0077e-05,\n        5.9531e-05, 4.5883e-05, 6.5449e-05, 4.9554e-05, 9.8446e-08, 5.5766e-05,\n        5.6535e-05, 1.1022e-04, 7.2413e-06, 2.2127e-05, 4.5780e-05, 3.7642e-05,\n        5.6162e-05, 1.2533e-05, 4.8463e-05, 7.3051e-07, 7.0802e-06, 8.7394e-05,\n        5.1142e-05, 9.3222e-05, 5.9558e-05, 5.3520e-05, 6.2056e-05, 5.1860e-05,\n        6.2345e-05, 1.1813e-04, 4.4469e-05, 6.1378e-05, 5.4675e-07, 6.0514e-07,\n        6.8865e-05, 1.3582e-04, 8.0553e-05, 4.9708e-05, 6.0015e-05, 6.1626e-05,\n        5.5857e-05, 6.6562e-05, 5.0500e-05, 1.0480e-05, 5.3138e-05, 6.2440e-05,\n        4.3975e-05, 4.3015e-05, 1.4075e-07, 4.1689e-05, 6.3768e-05, 3.6194e-05,\n        5.5089e-07, 5.1195e-05, 5.9280e-05, 4.4625e-08, 1.3426e-07, 6.4783e-05,\n        6.1671e-05, 7.4173e-05, 3.5884e-05, 7.4318e-05, 3.4199e-05, 6.2636e-05,\n        4.9733e-05, 3.5539e-05, 3.9190e-05, 6.2689e-05, 6.2631e-05, 5.4694e-05,\n        8.1090e-05, 7.6418e-05, 6.4501e-05, 3.5079e-05, 5.3659e-05, 1.6279e-04,\n        4.1786e-05, 5.2087e-05, 2.7329e-05, 5.4073e-05, 6.3129e-05, 9.9276e-05,\n        1.4463e-06, 4.7581e-05, 7.8885e-05, 3.6910e-05, 7.5648e-05, 5.6108e-05,\n        6.1684e-05, 4.9811e-05, 5.5668e-05, 8.1306e-05, 7.6103e-07, 6.9993e-08,\n        8.0110e-07, 5.5193e-05, 4.9125e-05, 6.7111e-05, 6.0407e-05, 1.5247e-06,\n        5.0229e-05, 5.9279e-05, 2.5739e-05, 5.7219e-05, 4.1998e-05, 4.4227e-05,\n        8.3390e-05, 6.9618e-05, 4.7138e-05, 9.0475e-05, 6.8649e-05, 8.6704e-05,\n        5.7291e-05, 6.2965e-05, 6.1046e-05, 6.3377e-05, 8.6306e-05, 8.9585e-05,\n        8.1385e-05, 4.0861e-05, 5.0476e-05, 5.1084e-05, 5.8064e-05, 4.2190e-05,\n        4.8913e-05, 6.1226e-05, 1.2306e-04, 9.7292e-05, 7.2088e-05, 7.9435e-05,\n        6.0723e-05, 2.1911e-05, 2.4702e-05, 6.3349e-05, 5.6102e-05, 5.9785e-05,\n        3.3363e-05, 3.4988e-08, 1.0017e-06, 7.3015e-07, 6.3292e-05, 4.8849e-05,\n        5.9466e-05, 7.0894e-05, 5.3603e-05, 4.3099e-05, 4.0995e-05, 7.9944e-05,\n        4.0672e-05, 2.5679e-07, 4.2538e-08, 7.4015e-05, 5.2913e-05, 8.9377e-05,\n        6.6951e-05, 5.6542e-05, 9.5307e-08, 4.0459e-05], device='cuda:0')"
       },
       "2": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([[ 3.0606e-06, -7.0439e-06, -9.3668e-06,  ...,  1.9523e-05,\n         -6.8146e-25,  3.6510e-07],\n        [-1.7220e-05, -8.9561e-06,  4.2140e-05,  ...,  6.9116e-05,\n         -5.2215e-25,  2.2923e-04],\n        [ 7.1131e-06,  2.5793e-05,  5.8272e-05,  ..., -9.9563e-06,\n         -6.9443e-25,  4.2099e-05],\n        ...,\n        [ 3.9873e-05,  6.0743e-06,  1.5934e-05,  ..., -4.3467e-06,\n         -9.9725e-25, -9.5190e-05],\n        [-4.2656e-05, -2.4305e-05, -5.0580e-05,  ...,  3.3702e-05,\n         -1.3243e-25, -4.7871e-05],\n        [-2.6036e-06, -3.1546e-05,  1.0162e-04,  ...,  7.3760e-05,\n          5.4446e-25, -1.6388e-04]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[9.3977e-09, 2.8265e-08, 2.0304e-08,  ..., 2.6207e-08, 1.8794e-13,\n         2.6969e-08],\n        [1.5989e-08, 6.8773e-08, 4.2667e-08,  ..., 5.1190e-08, 3.4671e-11,\n         4.2784e-08],\n        [1.3324e-08, 4.7240e-08, 3.7844e-08,  ..., 3.8673e-08, 2.3181e-10,\n         5.1588e-08],\n        ...,\n        [1.1073e-08, 5.1956e-08, 4.5597e-08,  ..., 4.7573e-08, 1.6871e-10,\n         5.4408e-08],\n        [2.2141e-08, 6.1669e-08, 4.6652e-08,  ..., 5.7102e-08, 1.5353e-12,\n         1.2886e-07],\n        [1.5296e-08, 5.8622e-08, 4.2407e-08,  ..., 7.2614e-08, 2.1330e-10,\n         3.7079e-08]], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[-7.9560e-06,  2.3993e-05,  3.2534e-06,  ...,  5.1585e-06,\n         -3.0564e-33,  5.7035e-06],\n        [-2.3746e-05, -4.0098e-05, -1.1183e-05,  ...,  1.7490e-05,\n         -2.7885e-33, -3.1322e-05],\n        [-2.1273e-05, -1.0974e-05,  1.0933e-06,  ...,  1.0596e-05,\n         -3.1965e-33,  4.0670e-05],\n        ...,\n        [-8.9289e-06,  5.3523e-05, -6.5075e-06,  ...,  2.6585e-05,\n          4.1366e-34,  2.6323e-05],\n        [-2.7970e-06,  3.2530e-05,  6.1558e-06,  ...,  4.4867e-05,\n          7.4581e-33,  1.1290e-04],\n        [ 8.1028e-06,  1.2931e-05,  5.9366e-08,  ...,  1.4795e-05,\n          5.9547e-33,  5.4140e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[4.0505e-09, 1.4402e-08, 1.2323e-08,  ..., 1.4270e-08, 5.3745e-14,\n         1.2518e-08],\n        [8.1800e-09, 3.9253e-08, 2.2590e-08,  ..., 2.9301e-08, 9.9092e-12,\n         2.1253e-08],\n        [6.0028e-09, 2.6590e-08, 2.3240e-08,  ..., 2.0730e-08, 6.6242e-11,\n         2.8640e-08],\n        ...,\n        [6.7921e-09, 2.7842e-08, 2.6969e-08,  ..., 2.5708e-08, 4.8211e-11,\n         3.2653e-08],\n        [9.4379e-09, 3.2343e-08, 2.5039e-08,  ..., 3.0256e-08, 4.4243e-13,\n         7.4594e-08],\n        [8.0911e-09, 3.1045e-08, 2.6019e-08,  ..., 3.9526e-08, 6.0953e-11,\n         1.9206e-08]], device='cuda:0')"
       }
     },
     "param_groups": [
       {
-        "lr": 0.00904518046337755,
+        "lr": 0.00793913236883622,
         "name": "scale_256",
         "betas": [
           0.9,
@@ -43,7 +43,7 @@
         ]
       },
       {
-        "lr": 0.00904518046337755,
+        "lr": 0.00793913236883622,
         "name": "scale_512",
         "betas": [
           0.9,
@@ -66,7 +66,7 @@
         ]
       },
       {
-        "lr": 0.00904518046337755,
+        "lr": 0.00793913236883622,
         "name": "scale_768",
         "betas": [
           0.9,
@@ -89,7 +89,7 @@
         ]
       },
       {
-        "lr": 0.00904518046337755,
+        "lr": 0.00793913236883622,
         "name": "scale_1024",
         "betas": [
           0.9,
@@ -112,7 +112,7 @@
         ]
       },
       {
-        "lr": 0.00904518046337755,
+        "lr": 0.00793913236883622,
         "name": "scale_1280",
         "betas": [
           0.9,
@@ -135,7 +135,7 @@
         ]
       },
       {
-        "lr": 0.004522637977440181,
+        "lr": 0.003969669238105037,
         "name": "fusion",
         "betas": [
           0.9,
@@ -195,7 +195,7 @@
     "T_i": 10,
     "T_mult": 2,
     "eta_min": 1e-06,
-    "T_cur": 2,
+    "T_cur": 3,
     "base_lrs": [
       0.01,
       0.01,
@@ -204,24 +204,24 @@
       0.01,
       0.005
     ],
-    "last_epoch": 2,
+    "last_epoch": 3,
     "_step_count": 0,
     "_is_initial": false,
     "_get_lr_called_within_step": false,
     "_last_lr": [
-      0.00904518046337755,
-      0.00904518046337755,
-      0.00904518046337755,
-      0.00904518046337755,
-      0.00904518046337755,
-      0.004522637977440181
+      0.00793913236883622,
+      0.00793913236883622,
+      0.00793913236883622,
+      0.00793913236883622,
+      0.00793913236883622,
+      0.003969669238105037
     ]
   },
   "metrics": {
-    "best_val_acc": 73.398,
-    "best_epoch": 1,
+    "best_val_acc": 74.29,
+    "best_epoch": 2,
     "scale_accuracies": {
-      "256": 73.398
+      "256": 74.29
     }
   },
   "train_config": {