Upload weights and configs - Run 20251012_032356

weights/best_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:44284136131c7abd3beeed57400f92b000b3208e86947fc46472887cea412e75
+oid sha256:cc187ab3f8538420f8a3440dd736f259e0c66a6587c78a5bbd13699917e2d8d9
 size 59515088

weights/best_model_metadata.json

@@ -1,26 +1,26 @@
 {
-  "epoch": 0,
+  "epoch": 1,
   "optimizer_state_dict": {
     "state": {
       "0": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([[ 2.2889e-05,  3.2026e-05,  1.6571e-06,  ...,  1.7219e-05,\n          1.1876e-05, -3.7499e-05],\n        [-2.1468e-04, -1.3604e-04,  1.3280e-04,  ..., -4.3560e-05,\n          2.0309e-04, -2.6094e-04],\n        [-6.7956e-05,  7.5952e-05,  9.7297e-05,  ..., -4.7797e-05,\n          2.9834e-05, -1.0805e-04],\n        ...,\n        [-1.1135e-04,  1.5752e-04, -1.0688e-04,  ...,  2.3727e-05,\n         -6.9641e-05,  1.9441e-05],\n        [-1.4223e-19,  4.0510e-20, -7.5370e-20,  ...,  1.2587e-19,\n         -9.1474e-20, -3.3843e-19],\n        [-1.0722e-05, -2.7671e-05,  1.7630e-05,  ...,  1.6293e-05,\n         -1.6963e-05, -1.2034e-05]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[1.1735e-07, 1.2837e-07, 1.1420e-08,  ..., 2.4014e-08, 2.3699e-08,\n         6.9322e-09],\n        [2.2943e-07, 1.5274e-07, 4.7403e-08,  ..., 9.2499e-08, 5.6044e-08,\n         6.6360e-08],\n        [2.0946e-07, 3.0776e-07, 6.8965e-08,  ..., 7.5621e-08, 5.2727e-08,\n         5.3820e-08],\n        ...,\n        [5.5617e-07, 3.9251e-07, 6.7908e-08,  ..., 6.5855e-08, 5.9912e-08,\n         4.2784e-08],\n        [2.1465e-10, 4.9741e-09, 7.8046e-10,  ..., 3.2963e-10, 1.0450e-09,\n         6.2219e-10],\n        [3.6456e-07, 2.1370e-07, 3.8428e-08,  ..., 3.7118e-08, 7.7678e-08,\n         4.0870e-08]], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([[-2.9847e-05, -1.5561e-05, -3.2015e-05,  ..., -3.4633e-05,\n          9.6170e-05, -4.8952e-05],\n        [-5.1054e-05, -3.7014e-05,  2.2638e-05,  ..., -1.1228e-04,\n         -1.0936e-04,  3.1119e-05],\n        [ 2.6958e-06, -1.2456e-04,  1.4858e-05,  ..., -4.4155e-05,\n          6.4120e-05,  2.8139e-05],\n        ...,\n        [-5.4542e-05,  2.4455e-05, -1.6261e-05,  ..., -3.6784e-05,\n          4.9569e-05, -5.8800e-05],\n        [-4.4965e-23,  2.3582e-24,  2.9657e-23,  ..., -1.0940e-23,\n          6.9415e-24, -2.3902e-23],\n        [-5.8117e-05,  2.1739e-05, -4.2380e-05,  ..., -5.3147e-05,\n          3.0465e-05,  8.0894e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[8.1019e-08, 1.0675e-07, 1.2200e-08,  ..., 2.7520e-08, 2.9280e-08,\n         8.1349e-09],\n        [2.0122e-07, 1.4826e-07, 4.4990e-08,  ..., 8.7383e-08, 4.9043e-08,\n         6.1986e-08],\n        [1.6635e-07, 2.2215e-07, 4.7403e-08,  ..., 5.8297e-08, 3.8775e-08,\n         4.2303e-08],\n        ...,\n        [4.0419e-07, 2.7279e-07, 4.6405e-08,  ..., 4.8686e-08, 4.1250e-08,\n         2.8963e-08],\n        [6.1337e-11, 1.4214e-09, 2.2304e-10,  ..., 9.4234e-11, 2.9866e-10,\n         1.7780e-10],\n        [2.7880e-07, 1.7346e-07, 2.7628e-08,  ..., 3.3208e-08, 6.5641e-08,\n         3.2027e-08]], device='cuda:0')"
       },
       "1": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([-6.0069e-04, -3.3064e-03, -3.0734e-03, -8.6551e-04, -1.1617e-13,\n        -1.9131e-04, -3.1353e-04,  1.9033e-03,  4.3359e-05, -9.1803e-04,\n         1.1774e-03,  9.3017e-04,  5.6052e-45,  3.8611e-03, -8.6159e-03,\n        -9.7585e-04, -7.5239e-04, -4.9659e-03,  4.8121e-04,  5.6052e-45,\n        -6.3107e-05, -2.2051e-03,  1.0329e-03, -1.9065e-03,  8.0956e-34,\n         4.0895e-03,  4.9298e-03, -8.0805e-04,  2.4687e-03, -1.3231e-03,\n        -1.3785e-03, -3.6809e-03, -1.2147e-07,  5.6052e-45,  1.2894e-03,\n        -5.8629e-05, -4.0794e-04, -1.3202e-05, -1.8225e-04,  2.7894e-03,\n         4.3474e-03, -1.8189e-03,  3.7872e-03, -6.5277e-04,  6.2215e-04,\n        -1.5357e-03,  3.2316e-04, -3.0323e-03,  4.6631e-03, -1.0720e-03,\n        -7.0228e-03,  5.6052e-45,  7.1771e-35,  5.6052e-45,  2.2352e-04,\n         7.5579e-04, -4.4723e-03, -2.0540e-03,  1.3815e-21,  1.4974e-04,\n        -4.9958e-03, -2.7003e-03,  3.7906e-03, -6.1028e-05,  2.5707e-03,\n         3.8245e-04,  2.5648e-03,  5.6052e-45,  2.3466e-03, -1.9835e-04,\n         2.0861e-08,  3.8633e-08,  3.9210e-03, -4.4422e-03,  1.3143e-03,\n        -1.9418e-03, -7.5711e-04,  4.5668e-03,  3.5378e-04, -2.4114e-03,\n         1.4986e-03, -4.5153e-03, -7.8498e-05,  1.9901e-04,  4.5906e-03,\n         2.3016e-03,  1.9876e-04,  9.5623e-04, -2.3304e-03,  1.5096e-03,\n         5.6052e-45,  1.7155e-03,  2.4380e-03, -1.3041e-03, -1.4891e-03,\n         1.5806e-03, -3.3790e-03, -7.7564e-05, -2.7520e-03,  4.8316e-04,\n        -7.0963e-04,  1.8868e-10, -2.4407e-03, -6.3466e-04, -6.3611e-03,\n         3.7107e-04, -2.2045e-03, -2.7769e-03, -1.1518e-05, -4.6362e-03,\n         2.2827e-03,  7.4211e-04,  5.6052e-45, -2.7134e-03,  1.6550e-09,\n         1.1651e-03,  1.1698e-03,  3.4867e-03,  5.6052e-45,  3.0735e-08,\n         9.0221e-10,  2.0252e-03,  5.6052e-45,  4.1098e-04,  2.3577e-04,\n        -2.5428e-04, -6.5745e-05,  5.6052e-45,  1.6520e-03, -1.0316e-03,\n         9.3832e-05, -1.2079e-04,  1.5227e-03,  7.9433e-15,  7.9189e-04,\n        -7.9527e-04,  1.5553e-03, -5.0773e-04,  7.7019e-04, -1.1426e-03,\n         5.1381e-03, -7.1637e-04, -1.8527e-03,  2.5450e-03,  8.9379e-03,\n        -3.2705e-03, -1.6546e-13,  4.0166e-03, -1.0601e-03,  6.1481e-04,\n         1.8809e-13, -6.7260e-04,  1.1489e-03,  8.6132e-39, -1.4846e-03,\n        -4.7213e-04,  2.3243e-23, -9.3133e-04,  2.6049e-03,  2.1563e-03,\n         3.9112e-31, -3.6412e-03, -5.2335e-03, -5.9905e-03, -2.3537e-03,\n        -1.3842e-03, -5.9578e-03, -2.2822e-03,  1.1170e-04, -2.1091e-29,\n         5.6052e-45,  3.0544e-03, -5.9425e-05, -4.0994e-03, -2.8030e-03,\n        -4.9734e-03, -1.4584e-03,  1.4601e-04,  4.9706e-34,  1.7165e-03,\n         2.3065e-03,  3.6659e-03,  5.6655e-04,  5.6052e-45,  3.5381e-03,\n        -2.2904e-03,  4.5980e-03,  2.6798e-03,  4.0761e-03,  5.6052e-45,\n         5.6052e-45,  5.0720e-04, -3.7422e-04,  1.9530e-03, -1.2886e-03,\n        -4.0029e-03,  2.1770e-03,  4.1103e-04,  5.6052e-45, -7.1649e-04,\n         7.2740e-04, -1.3782e-03,  2.7222e-03, -5.9347e-04, -1.2909e-03,\n         5.6052e-45, -8.0836e-03,  1.2741e-03, -4.9838e-04,  2.9652e-03,\n         4.9207e-03,  2.5242e-03,  4.6802e-03,  1.4867e-04,  3.1299e-04,\n        -2.2740e-03, -2.9689e-03, -5.3209e-04, -2.3417e-03,  1.3715e-04,\n         2.0833e-03, -5.4587e-03, -6.8023e-04,  6.9832e-04, -5.9483e-03,\n         3.2933e-03, -1.6839e-03, -4.2018e-03, -5.3849e-03,  2.9948e-03,\n        -4.8168e-03,  3.8610e-03, -1.4261e-03,  3.7585e-09, -3.0871e-03,\n        -9.9207e-04,  2.6115e-03, -4.9670e-03, -4.3795e-03,  3.4741e-03,\n         4.4612e-04, -2.7896e-03,  5.7169e-04,  4.8785e-03, -9.7763e-04,\n         3.7314e-03,  2.6435e-03,  1.3079e-03,  2.1492e-03, -4.0522e-03,\n         9.8097e-04, -3.8514e-03,  2.0170e-05, -1.6691e-03, -3.1188e-03,\n        -4.4586e-03,  5.6052e-45,  3.6943e-03, -1.2492e-03,  5.2881e-05,\n        -7.5173e-04,  1.7691e-03,  7.7236e-03,  1.0483e-03,  4.0832e-04,\n         5.3189e-04,  2.8803e-03,  5.6052e-45,  3.9974e-03,  2.7963e-04,\n        -4.9529e-04, -1.2284e-03,  9.8050e-04,  8.9134e-04, -3.3321e-03,\n        -2.5986e-17,  3.7972e-03,  5.6052e-45, -2.1032e-03,  7.8734e-04,\n         1.9879e-03,  1.1265e-03, -3.4019e-03, -3.1301e-03,  3.3404e-03,\n        -9.8025e-04,  5.6052e-45, -8.3202e-04,  7.9056e-04,  8.0925e-04,\n        -6.3958e-13, -2.7826e-03,  1.3710e-03,  3.5263e-03,  1.6505e-03,\n         4.7166e-04,  3.2478e-03,  6.0789e-04, -1.5110e-03, -3.8444e-03,\n        -3.0352e-04, -3.0284e-04,  2.6331e-08, -2.5967e-03, -1.9465e-03,\n        -9.0765e-12,  3.8184e-03, -1.1051e-09,  5.6052e-45, -3.7817e-03,\n        -3.6641e-03,  1.0406e-03,  1.8360e-03,  6.0505e-03,  1.2501e-03,\n         1.1163e-03,  8.2670e-04, -1.5502e-03,  1.2199e-03, -1.8529e-04,\n         1.3050e-04,  6.3554e-03, -6.4259e-03, -2.6184e-03, -1.9043e-03,\n         7.6206e-04, -4.6475e-03,  1.2827e-25,  6.9756e-03, -3.8316e-04,\n         3.1803e-05,  2.8865e-04,  5.3987e-03,  5.5536e-04,  5.6052e-45,\n        -5.6030e-04,  7.8950e-04, -9.7610e-05, -1.3669e-03,  1.2413e-03,\n         1.1517e-03, -2.8729e-04,  2.8656e-03, -2.8236e-03, -4.2717e-03,\n        -2.1533e-03,  5.6052e-45, -1.4698e-12,  9.4901e-04, -6.9550e-04,\n         6.9772e-04, -4.4212e-03,  1.4552e-03,  1.2531e-03,  2.5615e-03,\n        -3.7819e-03,  1.2623e-04, -1.0779e-03, -1.3615e-03,  3.9590e-18,\n         8.6027e-05, -2.0218e-03,  1.9077e-03, -1.2890e-03,  5.5033e-04,\n         1.2863e-03,  4.8764e-04, -1.0874e-03, -7.3269e-04,  2.0846e-03,\n         2.3644e-03,  3.5509e-03, -9.4110e-04, -2.7191e-05,  9.8474e-04,\n         5.4307e-03, -3.7414e-03,  8.9703e-04,  5.6052e-45,  8.7415e-04,\n         1.3296e-08,  5.6052e-45, -6.3380e-04,  3.7691e-03,  3.5902e-03,\n        -1.0605e-04,  4.4580e-04,  6.7394e-03, -2.4415e-03, -7.3135e-04,\n         2.0872e-03, -2.1997e-03,  4.1108e-03, -4.3807e-09,  5.6052e-45,\n        -4.9648e-04,  2.8766e-03, -1.2014e-03, -2.8814e-03, -5.5966e-05,\n         2.5405e-03, -8.5971e-04, -5.4241e-06, -3.2462e-03,  3.1314e-04,\n        -2.5387e-03,  2.4806e-03,  1.4698e-33, -6.4468e-11, -6.2196e-03,\n         8.0881e-14, -6.5373e-10,  5.6052e-45, -1.3717e-03,  2.7646e-03,\n         3.4550e-03, -2.5083e-04,  5.6052e-45,  3.5205e-03,  1.3218e-03,\n         7.1738e-04, -3.0211e-04,  6.9750e-03,  1.2837e-03, -6.4541e-04,\n        -3.0453e-33, -3.1030e-03, -8.0829e-13,  2.2259e-03,  2.4388e-19,\n         1.5958e-03,  2.9320e-03,  7.4004e-05,  5.6052e-45, -4.8786e-03,\n        -9.6536e-04, -1.1047e-03,  3.5034e-04,  9.7487e-04,  3.5561e-03,\n        -1.8258e-03,  1.1405e-03, -4.4571e-03,  5.3981e-04, -1.9415e-03,\n         3.7651e-07, -2.9522e-03,  1.2595e-11, -3.1325e-04, -1.0205e-02,\n         5.6052e-45, -6.9883e-04, -3.1635e-04,  2.1702e-05, -2.1005e-02,\n         1.3513e-10,  7.1958e-05,  2.0322e-04, -2.4389e-03,  5.6052e-45,\n         1.5596e-03,  3.4362e-03,  5.9835e-43, -2.6713e-03, -1.0566e-18,\n        -1.8087e-03, -6.2851e-04,  5.6052e-45, -1.2291e-04, -2.5857e-03,\n         1.1538e-03, -4.0428e-03,  5.9601e-04,  1.7805e-03,  7.0921e-04,\n        -2.7822e-04,  3.5518e-23,  2.7884e-03,  5.6052e-45, -1.5361e-06,\n         7.0726e-09,  1.6324e-06,  3.7290e-03,  9.7123e-05,  6.5938e-03,\n         2.3920e-03,  3.0546e-03,  5.1159e-04, -5.2809e-03, -6.4831e-03,\n         3.8676e-15, -1.4933e-04,  1.7249e-03, -2.0113e-03, -2.7728e-19,\n         2.6408e-06, -8.0329e-03,  7.7057e-04,  2.2606e-03, -1.2932e-06,\n         9.5648e-05,  5.6052e-45, -1.5404e-08, -3.5735e-03, -5.7265e-04,\n         2.7375e-03, -1.1931e-03,  4.6036e-04, -8.1512e-04, -1.6971e-03,\n         2.8026e-03,  6.1842e-03, -2.0997e-03, -3.4300e-03,  7.6167e-11,\n         1.2152e-18,  5.9221e-03, -1.5254e-03,  1.2840e-03,  1.4807e-03,\n         2.2137e-04,  2.3902e-03, -3.6905e-04,  3.7090e-04,  3.9848e-03,\n         8.4553e-06, -8.6029e-03,  6.3052e-03, -1.4264e-03,  1.7582e-03,\n         2.1909e-22, -7.4063e-04,  6.8185e-05, -1.7367e-04, -7.4308e-08,\n        -3.5554e-03,  2.0595e-03,  7.3922e-23,  5.6052e-45,  4.7111e-04,\n         6.7966e-03, -6.9490e-03, -3.4873e-03, -2.8326e-03,  3.1004e-03,\n        -4.4247e-04, -5.0340e-03,  3.9281e-03,  3.5720e-03, -3.4490e-03,\n         1.4723e-03,  1.3372e-03,  8.2852e-03, -5.1868e-04,  6.5427e-04,\n        -1.0040e-03,  1.9325e-04,  2.7792e-03,  3.1484e-03, -7.3690e-04,\n        -7.9488e-04, -5.2024e-03, -5.9283e-04, -1.8571e-04,  1.3465e-21,\n        -1.3919e-03,  3.1642e-04,  2.8527e-03,  5.5314e-03,  9.5748e-04,\n        -6.3823e-03,  1.1961e-03, -3.7229e-03,  3.6283e-03,  6.2414e-42,\n        -7.7902e-16,  5.6052e-45, -3.2896e-03,  3.8389e-03, -2.9575e-04,\n         3.8170e-03,  1.0278e-21,  2.5214e-03, -2.4962e-04,  2.4568e-03,\n         3.4345e-03,  1.0050e-03, -9.1621e-04,  3.8970e-04, -6.1147e-04,\n         1.3594e-03, -2.9058e-03,  2.1731e-03,  4.0486e-03,  4.8792e-04,\n         2.2176e-03,  2.4161e-03, -9.1977e-04,  3.5772e-04,  2.5839e-03,\n         3.2797e-03, -1.7886e-03,  1.6143e-03, -1.6899e-03,  1.8400e-03,\n         2.0042e-03,  1.4821e-03, -3.1434e-04,  1.6967e-03,  9.6830e-04,\n        -3.2342e-04, -5.9173e-05,  3.4689e-03,  4.8732e-05, -2.4224e-03,\n        -1.1371e-03, -7.5282e-03,  3.2003e-03,  3.0720e-04, -2.8621e-13,\n         5.2720e-31, -8.1575e-18,  4.8461e-04, -5.0260e-03, -1.4113e-03,\n        -8.9169e-04, -6.4226e-04, -1.5311e-03, -3.3189e-04,  2.9600e-03,\n        -3.4607e-03,  5.0965e-07, -1.7981e-18,  2.3618e-04, -5.6518e-03,\n         1.5403e-03,  2.7384e-03, -6.1458e-04, -5.2167e-18, -9.0002e-04],\n       device='cuda:0')",
-        "exp_avg_sq": "tensor([3.2507e-05, 1.1724e-04, 8.7974e-05, 6.7685e-05, 1.8961e-06, 1.6011e-05,\n        1.1788e-04, 1.1777e-04, 6.9844e-05, 1.1874e-04, 1.1513e-04, 1.9905e-04,\n        6.4615e-06, 1.0866e-04, 1.1703e-04, 1.2000e-04, 1.8659e-04, 1.9433e-04,\n        7.8171e-06, 1.6823e-06, 7.4718e-05, 1.0555e-04, 8.7074e-05, 1.1808e-04,\n        6.1408e-06, 1.7379e-04, 1.2058e-04, 1.6009e-04, 9.5392e-05, 1.6592e-04,\n        1.2516e-04, 1.7138e-04, 1.1935e-05, 1.0846e-06, 5.2729e-05, 1.1998e-04,\n        1.4447e-04, 9.2601e-05, 5.1950e-05, 1.2680e-04, 1.3692e-04, 4.7013e-05,\n        1.2441e-04, 1.0247e-04, 5.4981e-05, 1.3501e-04, 1.2635e-04, 2.1873e-04,\n        1.6533e-04, 2.4761e-05, 1.7014e-04, 6.4423e-07, 1.0454e-06, 9.7833e-07,\n        1.1844e-04, 2.2787e-04, 8.1590e-05, 8.1723e-05, 3.1593e-06, 1.8851e-04,\n        1.0411e-04, 8.9025e-05, 1.6391e-04, 6.4249e-05, 1.3726e-04, 1.9040e-04,\n        1.2166e-04, 1.1974e-06, 1.3068e-04, 1.3169e-04, 2.6010e-06, 4.8913e-06,\n        1.6314e-04, 8.2426e-05, 8.8617e-06, 1.0488e-04, 1.1895e-04, 1.6822e-04,\n        3.9032e-05, 1.2856e-04, 1.6754e-04, 1.1636e-04, 1.5123e-04, 9.2484e-05,\n        1.3653e-04, 7.5866e-05, 1.4074e-04, 3.2978e-05, 1.1580e-04, 9.9418e-05,\n        7.6357e-06, 1.8128e-04, 1.1747e-04, 5.9849e-05, 1.3040e-04, 2.3896e-05,\n        8.8402e-05, 4.6617e-05, 8.6008e-05, 6.9984e-05, 7.8474e-06, 6.2019e-06,\n        1.3756e-04, 1.5139e-04, 1.6610e-04, 1.4352e-05, 1.1485e-04, 1.7282e-04,\n        9.9013e-07, 2.9003e-05, 1.7810e-04, 2.0762e-04, 5.1625e-07, 9.3318e-05,\n        3.4231e-06, 6.3253e-05, 1.6599e-04, 1.6560e-04, 2.3018e-07, 9.7048e-07,\n        8.3732e-07, 9.4250e-05, 6.6675e-07, 2.6900e-04, 1.1154e-04, 1.9790e-04,\n        1.1683e-04, 1.2542e-07, 3.9549e-05, 1.9470e-04, 9.2241e-05, 7.9176e-05,\n        1.5936e-04, 2.2375e-07, 1.5429e-04, 9.7220e-05, 3.8196e-05, 1.1510e-04,\n        8.2425e-05, 1.2534e-04, 2.0527e-04, 1.5900e-04, 1.5641e-04, 1.2322e-04,\n        1.5682e-04, 1.1042e-04, 9.8737e-06, 1.4441e-04, 1.0458e-04, 8.0399e-05,\n        8.4686e-06, 8.8142e-05, 7.9128e-05, 4.5436e-06, 1.0553e-04, 1.3169e-04,\n        8.9994e-06, 1.6063e-04, 9.3803e-05, 1.4917e-04, 7.5638e-06, 1.2990e-04,\n        2.3577e-04, 9.4472e-05, 1.0972e-04, 9.9431e-05, 2.3060e-04, 7.3299e-05,\n        9.5476e-05, 1.0670e-06, 1.6478e-06, 1.2884e-04, 1.7710e-04, 1.1040e-04,\n        7.9592e-05, 1.5554e-04, 4.1986e-05, 1.4639e-04, 2.9905e-07, 3.7256e-05,\n        9.7225e-05, 1.7506e-04, 6.7457e-07, 2.2003e-06, 1.5172e-04, 1.2292e-04,\n        1.9601e-04, 5.9613e-05, 9.7429e-05, 2.9050e-07, 7.3207e-06, 1.1665e-04,\n        6.0249e-05, 1.2152e-04, 1.3849e-04, 1.1670e-04, 7.3939e-05, 1.3982e-04,\n        1.1539e-06, 1.3118e-04, 5.3802e-05, 5.9900e-05, 2.0751e-04, 3.6520e-05,\n        2.3494e-04, 5.6914e-08, 1.5068e-04, 1.3239e-04, 1.0836e-04, 1.5655e-04,\n        5.6113e-05, 1.1710e-04, 1.3372e-04, 7.8968e-05, 3.2596e-05, 1.1261e-04,\n        6.6086e-05, 1.4019e-04, 1.5585e-04, 6.9301e-05, 1.0123e-04, 1.4779e-04,\n        1.0782e-04, 8.9897e-05, 1.1035e-04, 8.6228e-05, 2.1523e-04, 1.7441e-04,\n        8.3166e-05, 1.8110e-04, 9.4618e-05, 2.0647e-04, 1.5496e-04, 1.4147e-06,\n        1.3028e-04, 6.9138e-05, 1.1595e-04, 9.8562e-05, 2.6164e-04, 9.7953e-05,\n        4.8600e-05, 5.7490e-05, 1.4495e-04, 1.5176e-04, 1.7491e-04, 8.0334e-05,\n        4.1977e-05, 8.6283e-05, 8.5547e-05, 1.2930e-04, 9.6704e-05, 1.1941e-04,\n        8.9328e-05, 1.0880e-04, 1.4305e-04, 1.2733e-04, 6.0956e-06, 1.0217e-04,\n        5.3809e-05, 3.1618e-06, 1.2085e-04, 1.9260e-04, 1.2522e-04, 1.3108e-04,\n        1.2611e-04, 1.4462e-04, 8.4020e-05, 9.0356e-07, 1.1884e-04, 1.5240e-04,\n        7.2876e-05, 1.6757e-04, 1.2694e-04, 6.7720e-06, 1.7029e-04, 1.5517e-06,\n        1.1217e-04, 4.4558e-06, 7.8680e-05, 3.5964e-06, 1.0988e-04, 7.8484e-05,\n        1.9183e-04, 1.3617e-04, 1.6142e-04, 1.3095e-04, 1.8173e-07, 1.1762e-04,\n        1.5120e-04, 7.8412e-05, 2.6337e-06, 1.1580e-04, 1.8360e-04, 1.4545e-04,\n        1.2320e-04, 1.4313e-04, 1.9809e-04, 1.0087e-04, 1.3083e-04, 1.9717e-04,\n        8.9043e-05, 4.9356e-06, 1.2835e-05, 7.3823e-05, 1.4438e-04, 3.8133e-05,\n        1.2614e-04, 1.8736e-07, 3.6950e-06, 1.2990e-04, 1.5428e-04, 8.8410e-05,\n        1.4186e-04, 1.4564e-04, 5.6568e-05, 1.4128e-04, 1.6564e-04, 1.4981e-04,\n        7.7269e-05, 1.4320e-04, 1.4481e-04, 2.0914e-04, 1.7764e-04, 7.9841e-05,\n        1.6716e-05, 1.5881e-04, 1.0352e-04, 3.2232e-06, 8.7772e-05, 1.5606e-04,\n        5.2200e-06, 6.3725e-05, 1.9099e-04, 9.9991e-05, 4.4081e-10, 9.8620e-05,\n        5.8329e-05, 1.7657e-04, 8.5530e-05, 1.0080e-04, 8.0978e-06, 1.4344e-04,\n        1.1621e-04, 1.0616e-04, 1.0820e-04, 1.6679e-04, 6.4509e-06, 1.5713e-05,\n        1.3086e-04, 1.3924e-04, 5.7230e-05, 1.7347e-04, 1.0459e-04, 1.2532e-04,\n        1.9473e-04, 7.8555e-05, 9.9183e-05, 1.0078e-04, 1.6287e-04, 6.1473e-06,\n        6.2829e-05, 6.3714e-05, 1.2282e-04, 1.1346e-04, 1.5332e-04, 2.7345e-04,\n        1.5856e-05, 1.2311e-04, 1.4564e-04, 1.7048e-04, 1.5353e-04, 1.3800e-04,\n        1.1477e-04, 2.7784e-06, 1.2321e-04, 2.1425e-04, 2.5506e-04, 1.9452e-05,\n        2.8083e-06, 1.9102e-04, 4.0009e-06, 1.5322e-05, 1.0637e-04, 2.2128e-04,\n        3.8706e-05, 9.9500e-05, 1.4568e-04, 2.1639e-04, 8.7405e-05, 6.0282e-05,\n        1.4450e-04, 1.3912e-04, 9.9623e-05, 1.5229e-06, 5.0533e-06, 1.6214e-04,\n        1.4565e-04, 1.5037e-04, 1.4981e-04, 6.1761e-05, 8.6326e-05, 9.5113e-05,\n        5.7161e-06, 7.1970e-05, 7.9266e-05, 1.4965e-04, 9.8810e-05, 1.8609e-06,\n        6.2232e-07, 2.6146e-05, 5.3609e-07, 6.4562e-06, 9.3529e-07, 1.8374e-04,\n        1.0017e-04, 2.0571e-05, 6.9488e-05, 2.5251e-06, 4.9152e-05, 1.0061e-04,\n        8.1306e-05, 1.1810e-04, 1.4575e-04, 2.0476e-04, 1.2326e-04, 6.5627e-06,\n        3.4647e-05, 1.9104e-06, 6.4540e-05, 3.5233e-07, 3.7474e-05, 1.2947e-04,\n        1.4519e-04, 1.6220e-06, 7.6871e-05, 1.1438e-04, 1.8536e-04, 1.7969e-04,\n        8.1910e-05, 2.3525e-05, 1.2165e-04, 4.8800e-05, 1.5248e-04, 1.6620e-04,\n        8.4353e-05, 4.3623e-06, 9.5107e-05, 5.4737e-07, 1.1433e-04, 6.0626e-05,\n        1.8163e-05, 1.6771e-05, 9.4185e-05, 5.5767e-06, 1.2817e-04, 8.5737e-07,\n        5.7967e-05, 2.3343e-06, 8.8746e-05, 2.4520e-05, 1.3919e-04, 1.5167e-04,\n        1.5942e-07, 1.0202e-04, 2.0640e-05, 1.5169e-04, 8.0813e-05, 1.3020e-06,\n        1.0727e-04, 7.4665e-05, 2.5416e-04, 1.8834e-04, 8.4547e-05, 1.4440e-06,\n        6.9902e-05, 1.4743e-04, 9.0501e-06, 1.4931e-04, 2.8734e-06, 5.9674e-06,\n        5.0801e-07, 8.4795e-06, 9.1691e-05, 1.5931e-04, 2.0231e-04, 1.4534e-04,\n        7.5968e-05, 1.0432e-04, 1.2185e-04, 9.2049e-05, 1.2056e-06, 1.2896e-04,\n        1.1794e-04, 2.2746e-04, 1.6985e-06, 5.9872e-06, 1.0515e-04, 8.1016e-05,\n        8.1447e-05, 7.6041e-06, 1.1137e-04, 8.9460e-06, 2.0996e-06, 1.9050e-04,\n        1.2343e-04, 1.7245e-04, 1.0255e-04, 1.0953e-04, 1.0919e-04, 5.8683e-05,\n        1.3136e-04, 2.1726e-04, 7.8684e-05, 1.3435e-04, 6.6953e-06, 7.4107e-06,\n        1.6702e-04, 2.5935e-04, 1.7216e-04, 1.1827e-04, 1.4945e-04, 1.4070e-04,\n        1.3730e-04, 1.0282e-04, 1.0497e-04, 2.0012e-05, 1.3467e-04, 1.4184e-04,\n        7.3608e-05, 7.6452e-05, 8.8413e-07, 6.9574e-05, 1.4512e-04, 6.6037e-05,\n        6.7463e-06, 1.1129e-04, 1.2146e-04, 5.4648e-07, 1.6442e-06, 1.2454e-04,\n        1.4420e-04, 1.5347e-04, 7.9989e-05, 1.4221e-04, 7.9842e-05, 1.4321e-04,\n        8.5637e-05, 6.5051e-05, 9.0108e-05, 1.1111e-04, 1.4506e-04, 1.4121e-04,\n        1.6361e-04, 1.2710e-04, 1.2343e-04, 3.8292e-05, 1.1329e-04, 2.9475e-04,\n        1.0576e-04, 9.4500e-05, 5.2959e-05, 1.2773e-04, 1.2505e-04, 2.0023e-04,\n        3.5716e-06, 9.7151e-05, 1.6800e-04, 5.4568e-05, 1.7418e-04, 1.2755e-04,\n        1.0674e-04, 1.2603e-04, 1.3099e-04, 1.8476e-04, 1.5367e-06, 8.5715e-07,\n        9.8105e-06, 1.3796e-04, 1.0924e-04, 1.4638e-04, 1.5233e-04, 1.8672e-05,\n        9.3651e-05, 1.0190e-04, 1.6308e-05, 1.0922e-04, 7.2146e-05, 9.3805e-05,\n        1.6084e-04, 1.4160e-04, 6.4953e-05, 1.3259e-04, 1.1345e-04, 1.7007e-04,\n        1.2662e-04, 7.4785e-05, 1.3804e-04, 1.3471e-04, 2.1403e-04, 1.7911e-04,\n        1.7954e-04, 7.4847e-05, 1.1543e-04, 9.6103e-05, 1.1780e-04, 9.8543e-05,\n        8.7148e-05, 1.3462e-04, 2.5042e-04, 1.8533e-04, 1.9444e-04, 1.8729e-04,\n        1.6371e-04, 3.8871e-05, 3.1880e-05, 1.4484e-04, 1.1660e-04, 1.3105e-04,\n        6.3615e-05, 3.2822e-07, 1.2267e-05, 9.3126e-07, 1.4859e-04, 1.1661e-04,\n        1.2378e-04, 1.9250e-04, 8.5557e-05, 7.5857e-05, 7.2122e-05, 1.3369e-04,\n        7.1680e-05, 2.9975e-06, 9.6274e-09, 1.7516e-04, 9.9866e-05, 1.9206e-04,\n        1.4256e-04, 1.1975e-04, 1.1671e-06, 8.0871e-05], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([ 1.7719e-03, -2.7156e-03,  1.7750e-03,  6.0193e-04, -1.2922e-15,\n         2.1121e-04,  4.9933e-04, -2.6053e-03,  3.6154e-04,  7.6070e-04,\n         2.2209e-04, -3.5991e-03,  5.6052e-45,  2.6509e-04,  5.7084e-04,\n         1.4721e-05, -4.8943e-04,  4.4611e-03,  7.7828e-04,  5.6052e-45,\n         1.3613e-04, -3.5591e-04,  3.9319e-04,  5.0281e-04,  5.6052e-45,\n         5.8405e-04, -2.1400e-04,  2.3399e-03,  3.3233e-03, -1.1443e-03,\n        -1.8175e-03,  1.7404e-03, -6.8076e-11,  5.6052e-45, -9.1142e-04,\n         1.8191e-04, -2.5487e-03,  2.2662e-03,  1.0145e-03, -1.1766e-03,\n        -2.9495e-04,  3.0875e-03,  1.2050e-04, -6.2883e-03, -2.0681e-04,\n        -1.0954e-03, -2.4520e-03, -7.5239e-05,  1.6260e-03,  1.2031e-03,\n        -8.9359e-04,  5.6052e-45,  4.0436e-08,  5.6052e-45,  9.4041e-04,\n        -8.9957e-03,  8.6776e-04,  1.6575e-03,  5.6052e-45,  2.1743e-03,\n        -2.7675e-03, -1.0951e-03, -1.4413e-03,  2.6270e-03,  3.0545e-03,\n        -5.3882e-04, -2.0457e-03,  5.6052e-45, -1.2317e-03, -2.6512e-03,\n        -1.2740e-04,  5.6052e-45,  3.3783e-03,  8.1476e-04,  7.9997e-04,\n        -1.2763e-05,  2.0720e-04, -1.9931e-03, -2.9498e-03, -4.0966e-05,\n         4.4024e-04,  2.7361e-04, -1.2984e-03,  2.0401e-03,  2.5797e-03,\n        -1.8919e-04, -2.6561e-03, -2.7024e-03,  6.2633e-04,  2.1254e-03,\n         5.6052e-45, -3.3968e-03, -1.4859e-03,  3.0709e-04,  3.1062e-04,\n        -1.3830e-03, -1.2511e-03,  1.5813e-04,  2.0279e-03,  1.6878e-03,\n         2.7028e-03, -1.5681e-05,  1.8112e-03,  2.0162e-03,  1.1955e-03,\n         3.8765e-04, -1.6990e-03, -3.1365e-04, -3.3308e-03,  6.5740e-04,\n         3.3836e-03,  1.7968e-03,  5.6052e-45,  9.1714e-04, -3.7225e-11,\n        -1.5972e-03,  3.3398e-03, -4.4637e-04, -3.6734e-19,  1.9321e-06,\n         5.6052e-45, -1.6496e-03,  5.6052e-45, -4.3220e-04, -1.1750e-05,\n        -2.6207e-04,  2.5364e-03,  5.6052e-45,  2.4874e-04, -3.8771e-04,\n         3.1076e-03, -9.5760e-04,  6.0460e-04,  1.1282e-19, -9.6913e-04,\n         1.8747e-03, -2.3665e-03,  5.5657e-03,  3.6068e-04,  8.4913e-05,\n        -3.0548e-05, -1.7480e-03, -1.6406e-03, -8.9434e-04, -1.3305e-04,\n        -3.8913e-03, -5.6052e-45,  2.3141e-03,  1.2096e-03,  1.1739e-03,\n         3.7447e-19,  3.2070e-04, -2.2073e-03,  5.6052e-45,  2.6995e-04,\n         2.6495e-03,  1.7450e-16, -1.6806e-03, -7.4609e-04, -1.5372e-03,\n        -1.7810e-03,  1.6644e-04, -7.2881e-03, -3.3304e-03, -5.3491e-04,\n        -2.6843e-04,  1.1663e-03,  6.3363e-04,  1.8992e-03, -4.1072e-35,\n         5.6052e-45, -7.0372e-03, -7.7995e-04,  6.9158e-05,  3.1582e-04,\n        -3.8041e-03,  2.7068e-03,  5.4373e-04,  5.6052e-45, -1.8535e-03,\n         2.0693e-03, -1.7930e-03, -7.1912e-04,  5.6052e-45, -1.6341e-03,\n        -2.0785e-03, -1.5169e-03, -4.0041e-03,  6.6253e-03, -6.6289e-21,\n         5.6052e-45, -1.3885e-03,  1.3406e-03,  1.6914e-03,  5.4886e-04,\n         1.6931e-04, -4.8455e-04, -3.7299e-04,  5.6052e-45,  3.2643e-04,\n         1.9032e-03,  1.2197e-03,  1.2259e-03,  1.8186e-03,  1.2542e-03,\n         5.6052e-45,  2.3774e-03, -2.8570e-03,  6.9884e-04,  1.1143e-03,\n        -1.3073e-04, -3.2651e-04, -1.5357e-03, -1.7092e-02, -8.1063e-04,\n         3.6122e-03,  7.0732e-04,  3.2542e-03, -4.1671e-04, -2.1162e-04,\n        -8.7467e-05,  3.9853e-03, -1.5770e-03,  1.9945e-03,  5.1694e-04,\n        -2.4642e-05,  1.9110e-03, -8.1750e-03, -4.4143e-03,  3.0264e-03,\n        -5.1558e-05,  3.2541e-03,  2.5530e-03,  5.6052e-45, -4.2672e-04,\n         2.4149e-03, -1.5908e-03,  1.2887e-03, -2.3173e-03,  5.9383e-04,\n         1.4040e-04, -1.5054e-03,  1.8201e-03, -1.0404e-03,  4.2278e-03,\n         1.0953e-03, -8.5257e-04, -2.6585e-04, -3.4092e-04, -1.9889e-03,\n         1.5783e-03,  4.9310e-03, -3.4958e-04, -2.0176e-03, -4.1653e-05,\n         2.1225e-03,  5.6052e-45, -1.2615e-03, -1.5187e-03, -6.9453e-05,\n        -1.2908e-03,  1.5175e-03,  3.8103e-04,  3.2755e-03, -7.3108e-04,\n         3.3495e-03,  2.6484e-03,  5.6052e-45,  1.3955e-04,  2.4664e-03,\n         3.0513e-05,  1.5540e-03,  2.7154e-03,  1.2035e-03, -1.2743e-04,\n         1.5686e-11, -2.2926e-03,  5.6052e-45, -1.1629e-03, -1.0426e-03,\n         4.4875e-03,  3.8890e-04, -1.2673e-03, -2.6926e-03,  5.8198e-04,\n        -7.9894e-04,  5.6052e-45, -1.2130e-03,  1.6454e-03, -7.5367e-03,\n        -1.0624e-05, -3.8889e-03,  1.2439e-03, -2.1511e-03, -1.6852e-03,\n         5.8289e-03, -2.4693e-03, -2.8744e-03,  1.4555e-03,  5.4154e-04,\n        -3.0575e-03,  2.6792e-04,  1.5759e-07, -2.3129e-03, -2.6765e-03,\n        -9.7535e-07, -7.1819e-04,  6.4001e-40,  5.6052e-45, -3.2492e-04,\n        -1.8484e-03,  5.2411e-04,  4.2317e-03, -6.3674e-04, -1.5397e-03,\n         5.6048e-04,  2.3906e-03, -2.8164e-03, -1.1538e-03,  1.4445e-03,\n         3.8265e-03, -9.7848e-04,  4.0255e-04, -8.5947e-04, -8.6640e-05,\n        -4.9727e-05,  1.1320e-03, -2.1908e-05,  3.3552e-03,  5.8964e-06,\n         4.2318e-04, -2.3774e-03,  4.0055e-03, -6.5512e-04,  5.6052e-45,\n         1.2157e-03,  6.3504e-04,  3.2873e-03, -6.5067e-04,  1.6978e-04,\n        -3.8350e-04, -6.3015e-04, -5.1111e-04,  6.1542e-04,  3.4668e-03,\n         8.5541e-03,  5.6052e-45,  7.0538e-04,  1.7820e-03,  1.1006e-04,\n        -6.4889e-04, -3.8285e-04,  4.7411e-04,  4.0403e-03, -1.2390e-04,\n         1.4954e-03,  4.5684e-04, -4.0117e-04, -2.5432e-03,  5.4513e-28,\n         1.0953e-03,  1.2679e-03, -7.2141e-03,  2.2937e-03,  1.0306e-03,\n        -1.4254e-03, -3.4962e-04, -3.8257e-04,  1.2993e-03, -1.0368e-04,\n         1.0880e-03,  2.0327e-03, -7.5284e-04,  7.0328e-04,  1.6928e-03,\n         1.1690e-03,  2.2969e-04, -1.1355e-03,  5.6052e-45,  5.6912e-04,\n         6.9722e-04,  4.9105e-07, -4.4864e-04,  1.5886e-03, -2.5353e-04,\n         1.1741e-03, -5.4882e-03,  4.8552e-04, -2.3815e-03, -4.0432e-04,\n         3.2380e-03,  1.9961e-03,  5.5193e-04, -1.7605e-33,  5.6052e-45,\n         2.6521e-03,  2.4160e-04,  2.9053e-03, -2.0351e-04, -1.7770e-03,\n         2.2448e-03, -1.8172e-03,  2.3603e-03, -9.3427e-04,  4.6545e-04,\n         2.4674e-03, -1.9138e-03,  5.6052e-45,  2.6908e-04,  6.4993e-05,\n         5.6052e-45, -1.0407e-07,  5.6052e-45,  1.9614e-03,  1.4860e-03,\n         1.0679e-04, -7.2474e-04,  5.6052e-45,  8.8939e-04, -2.2507e-03,\n        -2.0403e-04,  2.4861e-04,  1.5738e-04,  4.1969e-03, -9.0950e-04,\n        -4.3497e-07, -4.5330e-03, -4.3736e-16,  2.6728e-03, -3.4864e-12,\n         6.6541e-04, -3.8415e-03, -4.9617e-03,  5.6052e-45, -2.8454e-04,\n         3.2791e-03,  2.8651e-05,  5.0693e-03,  1.8879e-03,  1.4716e-03,\n        -4.9795e-03, -1.4910e-03, -2.4324e-03,  2.8446e-03, -1.5682e-03,\n        -2.8933e-09,  1.4629e-03,  9.7059e-20, -7.9898e-04,  8.4393e-04,\n         5.6052e-45, -1.2870e-03, -1.4444e-03, -3.7080e-04,  2.6747e-03,\n         7.7871e-10, -2.4555e-03, -1.6801e-04,  1.6547e-03,  5.6052e-45,\n         1.5095e-04,  8.6976e-04,  5.6052e-45, -1.3492e-03,  7.0371e-17,\n         1.9926e-03, -1.2434e-05,  5.6052e-45,  4.2391e-03,  1.6444e-03,\n        -2.2978e-04,  1.0726e-03,  2.7664e-03,  6.5306e-04,  3.4669e-03,\n         1.4955e-03,  5.6052e-45, -6.9197e-04,  5.6052e-45,  9.3500e-04,\n         5.7621e-04,  6.1615e-04, -1.7161e-03,  1.6078e-04,  5.0908e-03,\n        -1.7193e-03,  1.2305e-03, -1.9431e-03, -1.9954e-04,  8.6554e-04,\n         6.1657e-44,  3.6795e-03, -2.0908e-03,  4.7156e-04,  3.5966e-04,\n         5.7927e-04,  5.7433e-05, -2.0558e-03,  2.1328e-03, -1.0483e-03,\n         2.0550e-03,  5.6052e-45,  4.8020e-06,  3.4740e-03, -9.1481e-04,\n        -2.3186e-03, -3.0433e-03,  2.2122e-03,  1.7908e-03,  2.8714e-03,\n        -5.2433e-04, -6.5450e-04, -5.8112e-03, -6.1113e-04,  1.2991e-09,\n         5.6052e-45, -3.7407e-04, -8.6636e-04, -7.9024e-04, -1.2939e-03,\n         1.6390e-03, -6.4169e-04, -1.7501e-03,  2.5247e-03,  1.5380e-03,\n         5.4311e-04, -1.9798e-03, -1.3643e-03,  3.7215e-04, -3.3887e-04,\n         1.2315e-12, -6.6939e-04,  4.9933e-03, -3.3018e-03, -5.6052e-45,\n        -9.1870e-03, -2.3953e-03,  9.9825e-34,  5.6052e-45, -1.8366e-03,\n         4.0818e-03, -7.4626e-05,  1.8173e-03,  3.3083e-03, -2.8544e-04,\n        -2.2003e-05,  8.8995e-04, -2.6031e-03, -2.9603e-03,  1.7691e-03,\n         1.1856e-03, -1.0590e-03, -1.7087e-03, -4.7046e-03, -2.6489e-03,\n         9.6220e-04,  1.1795e-03, -6.1423e-03,  2.8327e-03, -1.9739e-04,\n         9.0532e-04, -2.3302e-03, -1.0570e-03,  2.9605e-03,  9.4562e-05,\n        -2.4800e-03, -1.9176e-03,  2.6401e-03, -6.7864e-04,  9.1049e-04,\n         2.5613e-03,  1.8557e-03,  2.7468e-03,  1.7356e-04,  3.0913e-07,\n        -1.6716e-09,  5.6052e-45, -3.9027e-03, -1.2063e-03,  3.5184e-03,\n         4.1062e-03,  4.3985e-22, -1.2497e-03,  4.5070e-04,  1.4553e-03,\n         3.4370e-03,  3.0550e-04,  2.5467e-03, -6.4545e-04, -2.6825e-03,\n         1.6909e-03, -1.3134e-03,  2.0747e-03, -2.3815e-03, -8.7160e-03,\n        -5.2320e-03,  1.6452e-03,  6.0151e-04, -1.9492e-04, -4.6760e-04,\n        -8.1648e-03, -2.6751e-03,  5.5363e-04, -9.6228e-05, -1.5683e-03,\n         4.8903e-04,  2.4498e-03, -2.3832e-03,  1.7399e-03,  1.2952e-03,\n         8.9709e-04, -2.6526e-04, -5.7862e-04,  4.5445e-04,  6.4842e-04,\n        -9.3869e-05, -8.8490e-04,  4.7145e-03,  7.8731e-04,  3.4545e-12,\n         3.1115e-13,  1.7237e-07, -4.2924e-04,  2.8269e-03,  1.4195e-03,\n        -4.3912e-03,  5.2732e-03, -9.6656e-04, -2.4521e-04,  2.0737e-03,\n        -2.2138e-03,  2.4862e-28,  6.3193e-39, -1.6646e-03,  4.4695e-03,\n         1.0923e-03, -1.2242e-03, -7.8823e-04, -5.8591e-22, -2.6610e-04],\n       device='cuda:0')",
+        "exp_avg_sq": "tensor([2.6185e-05, 1.0394e-04, 6.1769e-05, 4.7187e-05, 5.4188e-07, 1.1386e-05,\n        8.2271e-05, 8.5614e-05, 5.1013e-05, 8.3434e-05, 8.1024e-05, 1.4614e-04,\n        1.8464e-06, 7.5689e-05, 8.3315e-05, 8.7148e-05, 1.2984e-04, 1.1138e-04,\n        1.7976e-05, 4.8074e-07, 5.8047e-05, 6.4854e-05, 5.8102e-05, 8.5291e-05,\n        1.7548e-06, 1.3080e-04, 7.9323e-05, 1.1112e-04, 7.1867e-05, 1.0145e-04,\n        8.9637e-05, 1.0222e-04, 3.5326e-06, 3.0992e-07, 6.2278e-05, 8.4035e-05,\n        9.4529e-05, 6.0741e-05, 3.8722e-05, 8.5220e-05, 9.3767e-05, 4.0801e-05,\n        8.1706e-05, 6.9468e-05, 5.5784e-05, 9.4584e-05, 8.3837e-05, 1.4525e-04,\n        1.2973e-04, 3.2615e-05, 1.0640e-04, 1.8409e-07, 2.9872e-07, 2.7956e-07,\n        9.5597e-05, 1.5516e-04, 5.8881e-05, 5.9887e-05, 9.0280e-07, 1.1803e-04,\n        7.7816e-05, 7.0783e-05, 1.2705e-04, 4.5680e-05, 1.0234e-04, 1.2013e-04,\n        8.0419e-05, 3.4217e-07, 1.0126e-04, 8.9997e-05, 3.2717e-06, 1.3977e-06,\n        1.1270e-04, 6.5817e-05, 1.3604e-05, 8.7232e-05, 9.4244e-05, 1.0724e-04,\n        3.5342e-05, 9.8258e-05, 1.2847e-04, 7.9171e-05, 1.0172e-04, 6.4004e-05,\n        8.7273e-05, 6.1845e-05, 9.3616e-05, 3.6464e-05, 8.5587e-05, 7.1024e-05,\n        2.1820e-06, 1.1252e-04, 7.3912e-05, 5.1679e-05, 1.2169e-04, 2.4996e-05,\n        6.7809e-05, 4.7224e-05, 6.6435e-05, 5.3782e-05, 2.7094e-05, 1.7723e-06,\n        1.0651e-04, 1.1359e-04, 1.4858e-04, 2.8774e-05, 7.7910e-05, 1.2017e-04,\n        7.7663e-06, 3.1601e-05, 1.3684e-04, 1.3643e-04, 1.4752e-07, 6.1557e-05,\n        9.7825e-07, 5.3982e-05, 1.0140e-04, 1.3039e-04, 6.5776e-08, 3.2824e-07,\n        2.3927e-07, 6.9650e-05, 1.9053e-07, 1.7587e-04, 7.5585e-05, 1.2484e-04,\n        9.4132e-05, 3.5841e-08, 2.9235e-05, 1.1996e-04, 6.7474e-05, 6.5300e-05,\n        1.0678e-04, 6.3938e-08, 1.0664e-04, 8.1291e-05, 3.6267e-05, 8.1214e-05,\n        5.7613e-05, 8.0973e-05, 1.4920e-04, 9.8473e-05, 1.0549e-04, 8.5102e-05,\n        9.9494e-05, 8.1772e-05, 2.8215e-06, 9.8957e-05, 7.6632e-05, 6.0016e-05,\n        2.4200e-06, 5.4160e-05, 6.3808e-05, 1.2984e-06, 7.0577e-05, 1.0513e-04,\n        2.5717e-06, 1.1417e-04, 6.7006e-05, 9.2386e-05, 6.7913e-06, 8.7186e-05,\n        1.9947e-04, 7.3766e-05, 7.8754e-05, 6.3501e-05, 1.4718e-04, 5.0104e-05,\n        7.4427e-05, 3.0491e-07, 4.7086e-07, 9.1470e-05, 1.1657e-04, 6.9807e-05,\n        6.6640e-05, 1.1796e-04, 3.5256e-05, 9.9528e-05, 8.5455e-08, 3.3643e-05,\n        7.4284e-05, 1.1369e-04, 6.7078e-06, 6.2876e-07, 1.0280e-04, 9.3019e-05,\n        1.2655e-04, 4.8381e-05, 6.5508e-05, 8.3013e-08, 2.0920e-06, 8.1143e-05,\n        4.1950e-05, 8.5873e-05, 9.3965e-05, 7.8254e-05, 5.4772e-05, 1.0383e-04,\n        3.2973e-07, 7.5674e-05, 5.0118e-05, 4.2554e-05, 1.3158e-04, 3.2573e-05,\n        1.7879e-04, 1.6264e-08, 8.9628e-05, 9.3593e-05, 7.0992e-05, 1.0373e-04,\n        4.2838e-05, 8.6818e-05, 9.0367e-05, 8.9597e-05, 4.0722e-05, 7.4222e-05,\n        5.1326e-05, 1.0765e-04, 1.0529e-04, 4.9929e-05, 6.4792e-05, 9.1554e-05,\n        6.7870e-05, 7.0876e-05, 7.7211e-05, 7.0628e-05, 1.3262e-04, 1.4059e-04,\n        5.7287e-05, 1.2511e-04, 8.2398e-05, 1.4213e-04, 1.0192e-04, 4.0426e-07,\n        9.4356e-05, 4.8902e-05, 7.8567e-05, 6.1416e-05, 1.8999e-04, 7.6075e-05,\n        4.4971e-05, 6.0220e-05, 8.8060e-05, 8.6001e-05, 1.3341e-04, 5.4736e-05,\n        3.7458e-05, 8.9258e-05, 5.9941e-05, 8.1535e-05, 7.6555e-05, 7.9841e-05,\n        6.3156e-05, 7.4600e-05, 1.0078e-04, 8.0634e-05, 1.7419e-06, 6.4350e-05,\n        4.5434e-05, 1.1492e-05, 8.4327e-05, 1.2536e-04, 9.3495e-05, 9.0823e-05,\n        8.5464e-05, 9.2246e-05, 6.1173e-05, 2.5820e-07, 8.8737e-05, 9.7515e-05,\n        6.1668e-05, 1.1619e-04, 7.8469e-05, 2.3436e-05, 1.1286e-04, 4.5058e-07,\n        8.2334e-05, 1.2733e-06, 5.7528e-05, 1.5404e-05, 7.8400e-05, 6.4173e-05,\n        1.3407e-04, 1.0631e-04, 1.0481e-04, 8.8875e-05, 5.1930e-08, 9.9428e-05,\n        1.1994e-04, 5.9341e-05, 1.1065e-06, 8.9699e-05, 1.1248e-04, 9.9404e-05,\n        7.6766e-05, 1.1760e-04, 1.3545e-04, 7.7120e-05, 8.2723e-05, 1.5759e-04,\n        6.8220e-05, 5.4418e-06, 3.6677e-06, 6.0936e-05, 8.8757e-05, 1.0897e-05,\n        9.4508e-05, 5.3596e-08, 1.0559e-06, 8.7500e-05, 1.1486e-04, 5.9288e-05,\n        9.1286e-05, 1.0201e-04, 7.0026e-05, 9.0122e-05, 1.0238e-04, 8.5591e-05,\n        5.7840e-05, 9.6669e-05, 1.0544e-04, 1.4472e-04, 1.2356e-04, 5.6766e-05,\n        2.4556e-05, 9.6125e-05, 7.5443e-05, 9.2146e-07, 6.3154e-05, 1.0093e-04,\n        1.0514e-05, 5.2251e-05, 1.1961e-04, 9.2645e-05, 1.2597e-10, 7.3657e-05,\n        4.6807e-05, 1.2449e-04, 5.7429e-05, 7.0788e-05, 1.1652e-05, 9.9527e-05,\n        7.5082e-05, 7.2245e-05, 7.6273e-05, 1.3233e-04, 1.8434e-06, 4.5481e-06,\n        8.8133e-05, 9.6553e-05, 5.2322e-05, 1.0399e-04, 7.0010e-05, 8.0560e-05,\n        1.3993e-04, 6.5921e-05, 7.4668e-05, 7.2204e-05, 1.0716e-04, 1.7566e-06,\n        4.2006e-05, 5.0364e-05, 9.5045e-05, 1.1070e-04, 9.8440e-05, 2.0420e-04,\n        1.8978e-05, 9.6445e-05, 1.0712e-04, 1.0500e-04, 1.0441e-04, 8.0089e-05,\n        7.3783e-05, 4.5673e-06, 7.9015e-05, 1.3546e-04, 1.8708e-04, 1.9370e-05,\n        8.0249e-07, 1.0890e-04, 4.9397e-06, 4.3785e-06, 7.1149e-05, 1.5961e-04,\n        5.1601e-05, 6.7308e-05, 8.8343e-05, 1.2766e-04, 5.8176e-05, 4.6119e-05,\n        1.1389e-04, 1.0290e-04, 6.7217e-05, 4.3518e-07, 1.4440e-06, 9.5064e-05,\n        9.4383e-05, 9.7615e-05, 1.1845e-04, 5.2660e-05, 6.8345e-05, 6.1756e-05,\n        1.4186e-05, 5.7426e-05, 6.6783e-05, 1.2038e-04, 6.5310e-05, 5.3178e-07,\n        1.4157e-06, 2.1271e-05, 1.5319e-07, 1.8450e-06, 2.6727e-07, 1.1933e-04,\n        8.5703e-05, 2.4135e-05, 5.8756e-05, 7.2158e-07, 3.5566e-05, 7.8141e-05,\n        5.5834e-05, 9.3492e-05, 1.0075e-04, 1.3223e-04, 7.8316e-05, 1.8754e-06,\n        4.3172e-05, 5.4594e-07, 4.6574e-05, 1.0072e-07, 3.9667e-05, 9.7900e-05,\n        8.4505e-05, 4.6350e-07, 5.1821e-05, 7.8551e-05, 1.1745e-04, 1.1118e-04,\n        6.1050e-05, 3.2160e-05, 9.7376e-05, 4.0605e-05, 9.7365e-05, 1.2358e-04,\n        5.8186e-05, 1.2478e-06, 7.2198e-05, 1.5644e-07, 7.9856e-05, 4.7343e-05,\n        5.1903e-06, 2.6853e-05, 6.3829e-05, 5.3001e-06, 8.1505e-05, 2.4521e-07,\n        4.6788e-05, 5.6387e-06, 7.1344e-05, 7.0069e-06, 8.7032e-05, 1.2208e-04,\n        4.5556e-08, 7.1572e-05, 5.8981e-06, 9.8941e-05, 6.0820e-05, 3.7207e-07,\n        9.8656e-05, 5.5876e-05, 1.6058e-04, 1.2404e-04, 6.0918e-05, 1.4882e-05,\n        5.3910e-05, 8.8351e-05, 2.5861e-06, 1.0310e-04, 8.2110e-07, 6.7687e-06,\n        3.4286e-06, 3.0858e-06, 6.5671e-05, 9.9121e-05, 1.5407e-04, 8.6984e-05,\n        7.1729e-05, 6.9371e-05, 8.7477e-05, 6.8605e-05, 3.4451e-07, 8.3578e-05,\n        8.1260e-05, 1.5208e-04, 4.7259e-06, 2.0069e-05, 6.8325e-05, 5.7288e-05,\n        7.2662e-05, 6.0905e-06, 7.6995e-05, 2.5564e-06, 6.0025e-07, 1.2225e-04,\n        7.8866e-05, 1.3945e-04, 8.2698e-05, 7.7207e-05, 9.0479e-05, 6.0077e-05,\n        9.0126e-05, 1.6558e-04, 6.1826e-05, 8.6941e-05, 1.9133e-06, 2.1177e-06,\n        1.0096e-04, 1.9103e-04, 1.1656e-04, 7.5653e-05, 9.0906e-05, 9.2470e-05,\n        8.5884e-05, 8.2854e-05, 7.5234e-05, 1.4954e-05, 8.3693e-05, 9.7825e-05,\n        6.3007e-05, 5.4483e-05, 2.5304e-07, 5.8177e-05, 9.6390e-05, 5.2218e-05,\n        1.9278e-06, 7.9757e-05, 8.1680e-05, 1.5617e-07, 4.6986e-07, 9.3479e-05,\n        9.5239e-05, 1.0603e-04, 5.2933e-05, 1.0776e-04, 5.5441e-05, 9.0652e-05,\n        6.6375e-05, 4.8456e-05, 5.6797e-05, 8.7112e-05, 9.3751e-05, 9.1886e-05,\n        1.0527e-04, 1.0298e-04, 9.0333e-05, 4.2948e-05, 7.7361e-05, 2.2191e-04,\n        6.5279e-05, 7.3744e-05, 3.9408e-05, 7.9760e-05, 9.1075e-05, 1.4052e-04,\n        1.0290e-06, 6.9951e-05, 1.1078e-04, 4.7856e-05, 1.0981e-04, 8.3493e-05,\n        8.2707e-05, 8.2696e-05, 8.3315e-05, 1.2272e-04, 4.3915e-07, 2.4494e-07,\n        2.8034e-06, 8.8715e-05, 7.5586e-05, 1.0030e-04, 9.2822e-05, 5.3358e-06,\n        6.6156e-05, 8.0489e-05, 2.5207e-05, 7.7702e-05, 5.6717e-05, 6.4486e-05,\n        1.1964e-04, 9.9142e-05, 6.0166e-05, 1.1964e-04, 9.7669e-05, 1.2682e-04,\n        9.5410e-05, 7.4313e-05, 9.2821e-05, 9.2618e-05, 1.3747e-04, 1.4180e-04,\n        1.2441e-04, 5.9513e-05, 7.8540e-05, 7.1513e-05, 8.1343e-05, 6.1158e-05,\n        6.6370e-05, 8.8826e-05, 1.7511e-04, 1.3670e-04, 1.1294e-04, 1.2170e-04,\n        9.5316e-05, 2.8118e-05, 3.2750e-05, 9.4559e-05, 8.2394e-05, 8.8387e-05,\n        4.8002e-05, 9.3894e-08, 3.5055e-06, 2.6614e-07, 1.0201e-04, 7.1645e-05,\n        8.9279e-05, 1.1596e-04, 7.1520e-05, 5.8988e-05, 5.3306e-05, 1.1040e-04,\n        5.0714e-05, 8.9861e-07, 4.1866e-09, 1.1326e-04, 7.4199e-05, 1.3079e-04,\n        1.0399e-04, 8.0540e-05, 3.3352e-07, 6.1059e-05], device='cuda:0')"
       },
       "2": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([[ 6.2244e-06,  1.0123e-04,  2.6873e-06,  ..., -5.1794e-05,\n         -1.0963e-20,  9.0553e-05],\n        [-1.5695e-05,  3.0202e-04,  7.8999e-06,  ..., -4.3161e-05,\n         -5.0507e-21, -3.3274e-06],\n        [ 6.2648e-08,  3.1202e-05, -4.5084e-05,  ...,  1.3314e-04,\n         -8.1531e-21, -2.5778e-05],\n        ...,\n        [-2.5402e-05, -9.5713e-06,  8.4876e-05,  ..., -1.2755e-05,\n         -4.5398e-20, -9.3574e-05],\n        [-3.9413e-05, -2.8487e-04,  1.7388e-05,  ..., -4.5623e-05,\n          5.0067e-20, -8.3133e-05],\n        [-1.7686e-05, -4.0777e-04, -3.5612e-05,  ..., -2.5136e-05,\n         -1.9603e-20,  6.6686e-06]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[2.3771e-08, 5.6179e-08, 3.7151e-08,  ..., 4.8737e-08, 6.5373e-13,\n         6.5375e-08],\n        [3.1993e-08, 1.1027e-07, 8.7912e-08,  ..., 8.8702e-08, 1.2129e-10,\n         7.8001e-08],\n        [3.1502e-08, 8.7614e-08, 6.8821e-08,  ..., 7.2139e-08, 8.1122e-10,\n         9.5629e-08],\n        ...,\n        [1.6788e-08, 9.1308e-08, 8.2521e-08,  ..., 9.2985e-08, 5.9039e-10,\n         9.7403e-08],\n        [5.4341e-08, 1.2883e-07, 1.0161e-07,  ..., 1.1026e-07, 5.3329e-12,\n         2.4452e-07],\n        [2.9475e-08, 1.0801e-07, 7.2145e-08,  ..., 1.3073e-07, 7.4640e-10,\n         6.5312e-08]], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([[ 3.0606e-06, -7.0439e-06, -9.3668e-06,  ...,  1.9523e-05,\n         -6.8146e-25,  3.6510e-07],\n        [-1.7220e-05, -8.9561e-06,  4.2140e-05,  ...,  6.9116e-05,\n         -5.2215e-25,  2.2923e-04],\n        [ 7.1131e-06,  2.5793e-05,  5.8272e-05,  ..., -9.9563e-06,\n         -6.9443e-25,  4.2099e-05],\n        ...,\n        [ 3.9873e-05,  6.0743e-06,  1.5934e-05,  ..., -4.3467e-06,\n         -9.9725e-25, -9.5190e-05],\n        [-4.2656e-05, -2.4305e-05, -5.0580e-05,  ...,  3.3702e-05,\n         -1.3243e-25, -4.7871e-05],\n        [-2.6036e-06, -3.1546e-05,  1.0162e-04,  ...,  7.3760e-05,\n          5.4446e-25, -1.6388e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[9.3977e-09, 2.8265e-08, 2.0304e-08,  ..., 2.6207e-08, 1.8794e-13,\n         2.6969e-08],\n        [1.5989e-08, 6.8773e-08, 4.2667e-08,  ..., 5.1190e-08, 3.4671e-11,\n         4.2784e-08],\n        [1.3324e-08, 4.7240e-08, 3.7844e-08,  ..., 3.8673e-08, 2.3181e-10,\n         5.1588e-08],\n        ...,\n        [1.1073e-08, 5.1956e-08, 4.5597e-08,  ..., 4.7573e-08, 1.6871e-10,\n         5.4408e-08],\n        [2.2141e-08, 6.1669e-08, 4.6652e-08,  ..., 5.7102e-08, 1.5353e-12,\n         1.2886e-07],\n        [1.5296e-08, 5.8622e-08, 4.2407e-08,  ..., 7.2614e-08, 2.1330e-10,\n         3.7079e-08]], device='cuda:0')"
       }
     },
     "param_groups": [
       {
-        "lr": 0.00975530705321762,
+        "lr": 0.00904518046337755,
         "name": "scale_256",
         "betas": [
           0.9,
@@ -43,7 +43,7 @@
         ]
       },
       {
-        "lr": 0.00975530705321762,
+        "lr": 0.00904518046337755,
         "name": "scale_512",
         "betas": [
           0.9,
@@ -66,7 +66,7 @@
         ]
       },
       {
-        "lr": 0.00975530705321762,
+        "lr": 0.00904518046337755,
         "name": "scale_768",
         "betas": [
           0.9,
@@ -89,7 +89,7 @@
         ]
       },
       {
-        "lr": 0.00975530705321762,
+        "lr": 0.00904518046337755,
         "name": "scale_1024",
         "betas": [
           0.9,
@@ -112,7 +112,7 @@
         ]
       },
       {
-        "lr": 0.00975530705321762,
+        "lr": 0.00904518046337755,
         "name": "scale_1280",
         "betas": [
           0.9,
@@ -135,7 +135,7 @@
         ]
       },
       {
-        "lr": 0.004877665762479736,
+        "lr": 0.004522637977440181,
         "name": "fusion",
         "betas": [
           0.9,
@@ -195,7 +195,7 @@
     "T_i": 10,
     "T_mult": 2,
     "eta_min": 1e-06,
-    "T_cur": 1,
+    "T_cur": 2,
     "base_lrs": [
       0.01,
       0.01,
@@ -204,24 +204,24 @@
       0.01,
       0.005
     ],
-    "last_epoch": 1,
+    "last_epoch": 2,
     "_step_count": 0,
     "_is_initial": false,
     "_get_lr_called_within_step": false,
     "_last_lr": [
-      0.00975530705321762,
-      0.00975530705321762,
-      0.00975530705321762,
-      0.00975530705321762,
-      0.00975530705321762,
-      0.004877665762479736
+      0.00904518046337755,
+      0.00904518046337755,
+      0.00904518046337755,
+      0.00904518046337755,
+      0.00904518046337755,
+      0.004522637977440181
     ]
   },
   "metrics": {
-    "best_val_acc": 72.356,
-    "best_epoch": 0,
+    "best_val_acc": 73.398,
+    "best_epoch": 1,
     "scale_accuracies": {
-      "256": 72.356
+      "256": 73.398
     }
   },
   "train_config": {