Upload weights and configs - David-decoupled-deep_efficiency - Run 20251012_065325

weights/David-decoupled-deep_efficiency/20251012_065325/best_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3425cb0338623f6fc015b24b1a194885adcf9f1414df023ce016335821eda6ed
+oid sha256:2f37bf4247645adf0df795b72b8f62941acd742446a82d070425620418d2292d
 size 59515088

weights/David-decoupled-deep_efficiency/20251012_065325/best_model_metadata.json

@@ -1,26 +1,26 @@
 {
-  "epoch": 0,
+  "epoch": 1,
   "optimizer_state_dict": {
     "state": {
       "0": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([[ 1.2550e-03,  1.2115e-03,  1.5959e-03,  ..., -1.6975e-04,\n         -5.3335e-04,  5.6941e-04],\n        [-2.4448e-04, -1.6424e-03,  5.5239e-04,  ...,  5.3398e-04,\n         -7.1624e-05,  1.8894e-04],\n        [ 7.1728e-04, -2.3394e-03, -7.1050e-04,  ..., -3.6111e-04,\n          1.5078e-04, -4.7855e-04],\n        ...,\n        [ 3.3854e-04,  1.0135e-04,  3.6995e-05,  ...,  1.0520e-03,\n         -4.2994e-04, -6.9080e-04],\n        [-1.2733e-04,  1.2000e-04, -3.4674e-04,  ...,  2.2357e-05,\n         -4.8729e-04, -4.1808e-04],\n        [ 8.6614e-04, -1.4720e-03,  1.0884e-05,  ..., -5.3164e-04,\n         -1.6090e-04,  8.2763e-04]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[1.7440e-06, 1.4150e-05, 5.4506e-06,  ..., 1.8313e-06, 1.0527e-06,\n         1.7278e-06],\n        [2.8963e-06, 8.6853e-06, 7.2384e-06,  ..., 2.1670e-06, 8.5851e-07,\n         1.7223e-06],\n        [2.7283e-06, 1.6133e-05, 7.7697e-06,  ..., 1.7042e-06, 9.5496e-07,\n         1.5529e-06],\n        ...,\n        [3.4748e-06, 2.4317e-05, 9.6203e-06,  ..., 2.5059e-06, 1.8602e-06,\n         1.8482e-06],\n        [2.7982e-06, 2.0598e-05, 7.9887e-06,  ..., 1.9767e-06, 1.2406e-06,\n         1.6987e-06],\n        [2.8421e-06, 1.5328e-05, 6.3706e-06,  ..., 1.7342e-06, 1.2565e-06,\n         3.5651e-06]], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([[ 1.9903e-04, -2.0316e-03,  4.0099e-04,  ...,  3.9600e-04,\n         -7.7503e-05, -3.2745e-04],\n        [ 1.2035e-04, -5.4277e-04,  8.3932e-04,  ..., -6.4219e-05,\n         -1.1444e-04,  2.8184e-04],\n        [-6.7256e-05,  9.3359e-04, -1.2770e-04,  ...,  2.4264e-04,\n         -4.6260e-04, -2.6548e-04],\n        ...,\n        [-9.3872e-06, -3.3246e-04, -1.0106e-04,  ..., -6.8116e-05,\n         -1.0831e-05, -4.9642e-04],\n        [-1.2853e-03, -3.1639e-03,  1.6793e-03,  ...,  8.5221e-04,\n          2.0776e-04, -5.3517e-04],\n        [-6.0768e-04, -1.0425e-03,  2.5003e-04,  ...,  1.3259e-04,\n         -6.0654e-05, -1.6917e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[1.4933e-06, 1.3383e-05, 4.9550e-06,  ..., 2.0071e-06, 1.0758e-06,\n         1.6587e-06],\n        [3.3611e-06, 9.2764e-06, 9.0210e-06,  ..., 2.5916e-06, 9.1690e-07,\n         1.3613e-06],\n        [2.5602e-06, 1.6735e-05, 6.9803e-06,  ..., 1.5625e-06, 1.0142e-06,\n         1.3210e-06],\n        ...,\n        [4.2617e-06, 3.2869e-05, 9.4941e-06,  ..., 2.8932e-06, 1.9778e-06,\n         1.9807e-06],\n        [2.4394e-06, 2.4098e-05, 7.0105e-06,  ..., 1.8268e-06, 1.1096e-06,\n         1.5990e-06],\n        [2.7173e-06, 1.4306e-05, 5.9305e-06,  ..., 1.5879e-06, 1.2643e-06,\n         3.5216e-06]], device='cuda:0')"
       },
       "1": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([ 1.8009e-02, -1.2055e-02,  2.7145e-03, -1.6758e-02,  1.6555e-02,\n         1.7012e-02,  1.1092e-02,  7.6679e-03,  7.8798e-03, -1.3388e-02,\n        -3.0474e-02,  2.8771e-03,  2.0017e-02, -2.0977e-02, -3.8319e-03,\n         2.9875e-03, -1.2310e-02,  1.7611e-02, -1.3063e-02,  1.0401e-02,\n         2.6546e-02,  1.2547e-02,  1.3663e-02,  4.4227e-03, -1.2496e-02,\n         6.8799e-03, -2.2087e-03, -2.1662e-02, -2.2350e-03,  1.1611e-02,\n        -1.0951e-02,  4.4428e-03,  1.1623e-03, -3.0684e-02, -1.0608e-02,\n        -5.0114e-03, -5.7342e-03, -1.2348e-02,  2.6019e-02,  1.4020e-02,\n        -6.8601e-03, -2.3226e-02,  1.1235e-02, -4.8562e-03, -2.6157e-03,\n         3.6935e-02,  6.9772e-03,  2.3951e-02, -3.2578e-03, -1.4148e-03,\n        -1.2304e-02,  1.0087e-02,  2.9436e-02, -2.7288e-03, -8.4325e-04,\n         5.1557e-03,  3.9505e-03,  2.5469e-02,  1.3783e-02, -2.0297e-02,\n        -9.2767e-03, -4.6670e-03, -9.3133e-03,  5.4297e-03, -8.8350e-03,\n        -2.6751e-02, -3.0748e-03, -6.6989e-03,  1.5386e-02, -6.5599e-03,\n         2.1800e-02,  3.3901e-03,  2.5686e-03,  1.3580e-02,  5.9859e-04,\n        -7.5739e-03, -8.3229e-03,  9.0494e-05, -8.3604e-03,  1.2921e-02,\n        -8.1473e-03, -1.4523e-02,  6.4808e-03, -4.8208e-03,  1.1210e-02,\n        -1.4359e-04,  1.1516e-03, -3.1006e-02,  7.3174e-03, -1.3510e-02,\n        -1.5211e-02,  2.1296e-02, -3.9276e-02,  2.8915e-02,  2.8444e-03,\n         5.1065e-03,  5.6457e-03, -5.5019e-03,  5.3282e-03,  4.9732e-03,\n        -2.3964e-04, -7.3066e-03, -1.1436e-02, -6.1499e-03,  1.2361e-02,\n         3.3437e-03, -1.5065e-02,  1.5324e-02,  4.9378e-02,  4.6606e-02,\n         1.6920e-02, -5.0095e-03, -3.6758e-02,  2.1377e-02,  2.5126e-02,\n         2.0128e-02, -1.5394e-03,  9.2842e-04,  1.8524e-02, -8.9826e-03,\n        -1.5035e-02, -3.5969e-03, -4.3122e-03,  4.4255e-03,  1.7996e-02,\n        -4.8313e-03, -4.9239e-02,  2.6680e-02, -3.4048e-03, -2.1566e-03,\n        -1.6207e-02,  4.3072e-03, -4.0487e-02,  5.4757e-03,  3.3412e-03,\n         1.1162e-03,  9.2407e-04, -6.0526e-03, -3.2964e-03, -1.4770e-02,\n        -7.8817e-03, -7.5860e-03,  1.3434e-02,  1.2915e-02,  3.9440e-03,\n        -9.4360e-03, -9.2378e-03, -7.5047e-03,  2.0284e-02,  4.0419e-03,\n        -1.1572e-02, -1.2330e-02,  7.7172e-03,  1.0355e-02,  1.6092e-04,\n         1.6118e-02,  6.0107e-03, -2.0840e-03,  2.0762e-02, -2.7050e-04,\n         5.1150e-03, -1.7372e-02, -2.4789e-03, -1.6021e-02, -3.0399e-02,\n         2.6306e-03,  6.0285e-03,  8.0217e-03, -1.9861e-02,  3.4911e-03,\n         1.9099e-02,  2.4285e-03, -6.6915e-03,  3.0813e-03,  3.4050e-03,\n         6.7982e-03, -2.0185e-02, -1.4912e-02,  3.0863e-03, -2.0957e-03,\n        -3.6317e-03,  2.6924e-02, -1.8081e-02, -1.4930e-02,  3.8414e-03,\n         4.0331e-03,  9.8740e-03,  9.4222e-03,  3.3798e-03,  1.5064e-02,\n        -3.6598e-03,  3.0023e-02,  1.1229e-02,  3.6134e-02,  1.3062e-02,\n         2.0409e-03,  4.6545e-03,  1.5086e-03, -6.8445e-03, -6.5030e-04,\n        -1.6011e-02,  1.1755e-02,  3.9490e-03, -1.2201e-02,  4.0803e-03,\n         2.2787e-02,  1.9468e-02, -2.3012e-02,  6.5805e-03,  9.8080e-03,\n        -4.3520e-04, -5.0737e-03, -5.8197e-02,  3.1425e-02,  1.2917e-02,\n        -3.3413e-02,  6.3814e-03,  9.6621e-03,  2.0004e-02,  4.8994e-03,\n        -1.7408e-02, -2.0388e-02, -3.4400e-02,  2.0836e-02, -5.7105e-02,\n         8.0472e-05,  1.5410e-02, -7.4141e-03,  1.6885e-03,  7.7513e-04,\n        -5.8255e-03,  1.6696e-02, -1.1442e-02, -6.3912e-03, -3.9089e-03,\n         2.1926e-02,  1.1429e-02, -9.6134e-04,  2.0697e-02,  1.4698e-02,\n        -5.6729e-04,  2.9769e-02, -2.0280e-03, -1.9240e-03,  4.5567e-03,\n         4.4323e-03,  4.5838e-02,  8.4752e-03, -2.2261e-02,  5.2204e-04,\n        -2.8574e-03,  1.3147e-02, -1.6788e-02,  3.4454e-02, -5.2498e-03,\n        -7.4057e-03,  7.3400e-03,  2.0018e-03, -2.3883e-02,  5.5050e-03,\n         9.3501e-03, -2.0206e-03, -1.3407e-02,  1.4906e-02, -9.8785e-03,\n        -3.7573e-02, -1.8116e-03,  5.1310e-03, -3.3217e-02, -2.5437e-03,\n        -1.5280e-03,  4.0751e-03, -1.5084e-02,  1.6595e-02,  7.1239e-03,\n         1.0263e-02, -8.0941e-03,  2.7762e-03,  9.8862e-03, -9.2338e-03,\n         7.7230e-03, -9.9331e-03,  6.9278e-03, -1.2121e-02,  7.5401e-03,\n        -2.0133e-02,  6.3171e-03,  3.7797e-03, -1.0425e-02, -2.4791e-02,\n         9.6918e-03,  2.3616e-02,  1.6261e-02, -1.8818e-02, -2.2344e-02,\n         2.6562e-02, -1.4125e-03, -3.8157e-03, -1.2566e-02, -3.4085e-03,\n         1.1423e-02, -1.2238e-03, -1.3460e-03, -2.3187e-03, -4.5871e-03,\n         2.3984e-02,  1.7443e-02, -1.2309e-02, -1.2024e-02, -4.8948e-03,\n         8.2930e-03, -2.4854e-02, -9.4574e-03,  4.1499e-02,  4.4833e-02,\n        -1.9503e-03, -4.5678e-02,  1.6744e-02, -4.1256e-03,  4.1862e-03,\n         2.8343e-02,  1.0924e-02, -2.3952e-02, -4.6015e-03, -5.9190e-04,\n         1.0575e-02,  1.3219e-02,  2.2263e-02,  5.3352e-03,  3.0385e-03,\n         1.7792e-02,  6.9044e-04,  2.0487e-02,  4.8465e-03, -2.5085e-04,\n         1.3600e-02,  3.5050e-03, -7.9639e-03,  6.0240e-03,  4.9627e-04,\n         1.4779e-02, -5.3624e-03, -1.4033e-02, -1.7258e-03,  1.8842e-02,\n         3.4092e-03,  1.0612e-03,  3.2673e-03,  4.7491e-03,  6.5023e-03,\n        -2.5678e-02,  7.9316e-03,  3.6764e-02,  2.1623e-02, -6.4210e-03,\n         7.8400e-03,  8.0539e-03, -1.8145e-02,  1.2596e-02, -1.3035e-02,\n         5.4579e-03,  2.1838e-02, -9.4126e-04,  5.6897e-03, -6.7038e-03,\n        -3.2216e-02, -1.6959e-02, -3.0469e-03,  2.1368e-03, -1.0519e-03,\n         4.5414e-03, -1.0237e-03, -2.1151e-02, -2.8548e-02,  1.9635e-02,\n         1.3363e-02,  1.3576e-02, -1.3658e-02, -1.2032e-02,  7.7379e-03,\n         8.7361e-03, -1.3076e-02,  3.0275e-02,  1.9157e-02, -1.8404e-02,\n        -1.4202e-02,  8.0143e-03, -9.0673e-03,  1.3038e-02, -6.4873e-03,\n        -5.5454e-03, -3.9833e-03, -1.6290e-02, -8.3244e-03,  2.3857e-02,\n         1.4367e-02, -1.1359e-02,  6.5412e-04,  1.6699e-03,  2.9203e-02,\n        -3.2590e-03, -1.8969e-02, -2.8257e-02,  1.5945e-02, -2.0782e-02,\n         2.3899e-02,  3.0992e-02,  3.6734e-03, -1.1741e-02,  6.7889e-03,\n         9.4092e-03,  2.5447e-02,  2.9432e-02,  1.8043e-03,  3.8209e-03,\n         1.1499e-02,  1.8526e-03,  3.4454e-02, -9.8174e-03, -6.4240e-03,\n        -2.0281e-02,  8.4079e-03,  2.2863e-02,  4.8077e-03, -4.8680e-04,\n         9.3962e-03, -8.0193e-04,  2.7946e-02,  6.4275e-03,  2.3238e-02,\n         3.1635e-03, -1.5111e-02,  5.3614e-03, -1.7689e-02, -3.7519e-03,\n        -1.9930e-02, -8.0995e-04, -1.7285e-02,  1.7260e-02,  2.7992e-02,\n         1.3463e-03, -1.6507e-02, -3.5036e-03, -3.4247e-02, -5.5827e-03,\n         1.3767e-02,  2.3943e-02, -4.6461e-03,  2.1570e-02, -2.4197e-02,\n        -9.3189e-02,  1.0870e-02,  1.4933e-02,  2.3078e-02,  1.5114e-02,\n         6.3328e-03,  2.3941e-02,  1.0169e-02,  1.5573e-02,  1.8087e-02,\n         1.7939e-02,  1.0525e-02, -2.5594e-03, -1.0387e-02, -7.7410e-03,\n         1.3428e-02, -1.4518e-02,  1.0503e-02, -3.8376e-03,  2.4600e-02,\n         4.9708e-03, -6.0158e-02, -1.2168e-02,  1.6707e-02,  1.4549e-03,\n        -1.5190e-02,  3.4335e-03, -4.8629e-03, -3.3587e-03,  1.0559e-02,\n        -1.0010e-02,  1.0340e-02, -2.2399e-02, -2.6698e-02,  7.3545e-03,\n        -4.8817e-03,  1.1411e-02, -1.7723e-03, -7.1375e-03,  1.6514e-02,\n         1.4396e-02,  1.5428e-02,  4.4713e-03,  1.1762e-02,  1.7554e-05,\n         7.7379e-04, -4.2639e-03, -2.0993e-03,  3.5280e-04,  2.2307e-03,\n        -1.2941e-02,  4.3790e-03, -4.8959e-03,  1.1670e-02, -5.0001e-03,\n         1.2627e-02,  1.4449e-02, -1.8390e-02,  1.8667e-02, -1.7043e-02,\n        -8.4167e-03, -2.8037e-03, -1.4508e-02,  1.0449e-02, -2.0891e-02,\n         3.4059e-03,  4.4822e-02,  8.0777e-03,  9.8013e-03, -1.2572e-02,\n        -3.6712e-02, -8.2130e-03,  1.1418e-02, -1.8733e-02,  4.1356e-02,\n        -6.0896e-04, -6.1811e-03, -8.8506e-03,  1.9032e-02,  1.8583e-02,\n        -5.3788e-02, -1.7161e-02, -1.6044e-02, -3.6629e-03, -2.0540e-02,\n        -9.2696e-03,  1.3734e-02, -1.5894e-03,  2.0413e-02,  2.7401e-02,\n        -6.1899e-03,  2.0181e-02, -1.3870e-02, -2.3703e-02, -1.3949e-02,\n        -2.7396e-03,  2.9472e-02, -1.3818e-02,  3.8774e-04, -6.2968e-04,\n         5.8672e-03, -7.1886e-03,  8.9665e-03,  5.2106e-03, -4.0018e-02,\n         3.1149e-04, -1.4517e-02, -7.0883e-03,  3.1650e-03, -2.1674e-02,\n        -4.3195e-02,  3.1917e-03, -3.7221e-02, -6.1981e-04, -2.5183e-02,\n         1.3978e-02,  2.7227e-03,  1.6343e-02,  1.1705e-02, -2.9269e-02,\n         2.0101e-02, -4.6346e-02, -3.8836e-03,  9.6239e-03, -1.0911e-02,\n        -6.4772e-03,  3.9332e-03,  1.3184e-02,  1.9761e-03, -6.7081e-03,\n         1.5614e-03, -9.6779e-03, -1.3769e-02,  1.0621e-02,  1.1082e-02,\n         1.7812e-02,  2.2539e-02,  2.8735e-03,  3.1941e-03,  1.9327e-02,\n        -6.6690e-03, -2.0552e-02, -3.1661e-02, -2.7208e-03,  1.4585e-02,\n         1.6295e-02, -4.5207e-04,  1.2750e-02,  2.9023e-02,  1.3888e-02,\n         2.5544e-02, -9.7682e-03, -1.1103e-02,  2.6389e-03,  1.7108e-02,\n        -1.9441e-02, -2.2867e-02, -7.5663e-03, -1.4956e-04, -5.5262e-03,\n         8.3709e-03, -2.1896e-02,  1.6648e-02, -1.7241e-02, -1.6336e-02,\n        -8.0259e-03,  1.7809e-02,  4.3738e-03,  3.1967e-03,  4.7427e-03,\n         1.9254e-03,  9.1993e-03, -1.8402e-03,  4.0628e-03, -5.2144e-03,\n         8.3403e-03,  3.4274e-02,  9.9249e-03, -2.9123e-02, -2.5745e-03,\n         8.5084e-03, -5.7711e-03, -4.2764e-03,  1.9485e-02, -9.6255e-03,\n         4.7745e-03, -2.7050e-03, -1.6718e-02, -3.6621e-03,  1.5308e-02],\n       device='cuda:0')",
-        "exp_avg_sq": "tensor([0.0019, 0.0019, 0.0020, 0.0022, 0.0026, 0.0026, 0.0013, 0.0021, 0.0026,\n        0.0025, 0.0032, 0.0027, 0.0026, 0.0035, 0.0018, 0.0023, 0.0021, 0.0029,\n        0.0018, 0.0024, 0.0023, 0.0023, 0.0023, 0.0028, 0.0027, 0.0026, 0.0034,\n        0.0025, 0.0031, 0.0025, 0.0020, 0.0025, 0.0026, 0.0028, 0.0022, 0.0022,\n        0.0022, 0.0026, 0.0029, 0.0022, 0.0022, 0.0015, 0.0029, 0.0022, 0.0023,\n        0.0028, 0.0026, 0.0023, 0.0027, 0.0019, 0.0024, 0.0019, 0.0023, 0.0025,\n        0.0027, 0.0028, 0.0039, 0.0025, 0.0026, 0.0024, 0.0023, 0.0028, 0.0024,\n        0.0030, 0.0030, 0.0026, 0.0028, 0.0029, 0.0024, 0.0025, 0.0027, 0.0021,\n        0.0032, 0.0030, 0.0021, 0.0025, 0.0020, 0.0019, 0.0021, 0.0027, 0.0023,\n        0.0023, 0.0021, 0.0026, 0.0021, 0.0021, 0.0022, 0.0027, 0.0022, 0.0024,\n        0.0026, 0.0033, 0.0028, 0.0019, 0.0025, 0.0026, 0.0025, 0.0021, 0.0020,\n        0.0022, 0.0025, 0.0018, 0.0024, 0.0019, 0.0024, 0.0027, 0.0022, 0.0023,\n        0.0032, 0.0024, 0.0032, 0.0026, 0.0019, 0.0024, 0.0021, 0.0021, 0.0029,\n        0.0026, 0.0025, 0.0035, 0.0027, 0.0032, 0.0020, 0.0021, 0.0024, 0.0027,\n        0.0031, 0.0024, 0.0023, 0.0025, 0.0028, 0.0026, 0.0025, 0.0028, 0.0026,\n        0.0022, 0.0022, 0.0023, 0.0027, 0.0032, 0.0021, 0.0028, 0.0028, 0.0027,\n        0.0029, 0.0024, 0.0022, 0.0026, 0.0025, 0.0022, 0.0027, 0.0022, 0.0036,\n        0.0030, 0.0024, 0.0027, 0.0029, 0.0019, 0.0025, 0.0029, 0.0027, 0.0033,\n        0.0031, 0.0028, 0.0023, 0.0028, 0.0022, 0.0024, 0.0028, 0.0026, 0.0017,\n        0.0016, 0.0022, 0.0024, 0.0029, 0.0028, 0.0025, 0.0022, 0.0017, 0.0025,\n        0.0036, 0.0023, 0.0021, 0.0022, 0.0021, 0.0022, 0.0028, 0.0022, 0.0029,\n        0.0035, 0.0024, 0.0028, 0.0023, 0.0022, 0.0023, 0.0020, 0.0021, 0.0028,\n        0.0023, 0.0028, 0.0023, 0.0025, 0.0030, 0.0036, 0.0027, 0.0024, 0.0029,\n        0.0023, 0.0027, 0.0024, 0.0024, 0.0020, 0.0029, 0.0031, 0.0029, 0.0036,\n        0.0023, 0.0024, 0.0025, 0.0028, 0.0026, 0.0026, 0.0019, 0.0025, 0.0022,\n        0.0020, 0.0033, 0.0025, 0.0027, 0.0026, 0.0027, 0.0025, 0.0029, 0.0024,\n        0.0026, 0.0031, 0.0020, 0.0029, 0.0022, 0.0029, 0.0026, 0.0023, 0.0022,\n        0.0028, 0.0035, 0.0023, 0.0029, 0.0027, 0.0025, 0.0028, 0.0025, 0.0019,\n        0.0029, 0.0030, 0.0023, 0.0022, 0.0028, 0.0022, 0.0022, 0.0022, 0.0022,\n        0.0027, 0.0018, 0.0029, 0.0031, 0.0023, 0.0021, 0.0016, 0.0023, 0.0020,\n        0.0024, 0.0032, 0.0023, 0.0021, 0.0022, 0.0024, 0.0025, 0.0023, 0.0024,\n        0.0021, 0.0025, 0.0030, 0.0038, 0.0027, 0.0022, 0.0029, 0.0025, 0.0026,\n        0.0031, 0.0030, 0.0024, 0.0020, 0.0020, 0.0020, 0.0026, 0.0032, 0.0024,\n        0.0023, 0.0027, 0.0026, 0.0026, 0.0028, 0.0025, 0.0020, 0.0020, 0.0028,\n        0.0024, 0.0025, 0.0020, 0.0022, 0.0025, 0.0026, 0.0019, 0.0031, 0.0029,\n        0.0025, 0.0025, 0.0028, 0.0022, 0.0022, 0.0023, 0.0029, 0.0023, 0.0026,\n        0.0027, 0.0026, 0.0025, 0.0022, 0.0027, 0.0021, 0.0028, 0.0024, 0.0028,\n        0.0023, 0.0032, 0.0027, 0.0023, 0.0026, 0.0027, 0.0026, 0.0026, 0.0023,\n        0.0022, 0.0024, 0.0033, 0.0030, 0.0020, 0.0023, 0.0027, 0.0029, 0.0023,\n        0.0023, 0.0036, 0.0028, 0.0027, 0.0028, 0.0031, 0.0027, 0.0020, 0.0023,\n        0.0019, 0.0021, 0.0021, 0.0026, 0.0028, 0.0035, 0.0024, 0.0029, 0.0023,\n        0.0028, 0.0026, 0.0033, 0.0023, 0.0025, 0.0031, 0.0031, 0.0030, 0.0026,\n        0.0023, 0.0024, 0.0031, 0.0022, 0.0025, 0.0024, 0.0028, 0.0025, 0.0030,\n        0.0024, 0.0021, 0.0027, 0.0026, 0.0032, 0.0025, 0.0030, 0.0022, 0.0023,\n        0.0023, 0.0027, 0.0024, 0.0033, 0.0020, 0.0028, 0.0025, 0.0024, 0.0027,\n        0.0029, 0.0035, 0.0022, 0.0025, 0.0024, 0.0038, 0.0031, 0.0030, 0.0023,\n        0.0021, 0.0020, 0.0022, 0.0020, 0.0021, 0.0018, 0.0028, 0.0024, 0.0027,\n        0.0029, 0.0024, 0.0025, 0.0025, 0.0027, 0.0022, 0.0037, 0.0023, 0.0025,\n        0.0017, 0.0024, 0.0030, 0.0025, 0.0030, 0.0023, 0.0026, 0.0031, 0.0023,\n        0.0021, 0.0022, 0.0030, 0.0028, 0.0027, 0.0028, 0.0030, 0.0022, 0.0029,\n        0.0041, 0.0021, 0.0025, 0.0032, 0.0031, 0.0023, 0.0026, 0.0029, 0.0030,\n        0.0020, 0.0024, 0.0022, 0.0021, 0.0021, 0.0028, 0.0021, 0.0025, 0.0029,\n        0.0019, 0.0022, 0.0027, 0.0030, 0.0021, 0.0032, 0.0023, 0.0034, 0.0030,\n        0.0018, 0.0024, 0.0026, 0.0035, 0.0021, 0.0028, 0.0028, 0.0025, 0.0027,\n        0.0024, 0.0024, 0.0023, 0.0025, 0.0021, 0.0037, 0.0019, 0.0024, 0.0025,\n        0.0024, 0.0026, 0.0021, 0.0034, 0.0019, 0.0024, 0.0023, 0.0022, 0.0019,\n        0.0021, 0.0028, 0.0029, 0.0026, 0.0027, 0.0024, 0.0026, 0.0023, 0.0019,\n        0.0028, 0.0019, 0.0026, 0.0022, 0.0026, 0.0021, 0.0025, 0.0032, 0.0021,\n        0.0021, 0.0020, 0.0027, 0.0025, 0.0020, 0.0024, 0.0028, 0.0025, 0.0027,\n        0.0020, 0.0022, 0.0022, 0.0031, 0.0024, 0.0025, 0.0026, 0.0022, 0.0026,\n        0.0025, 0.0024, 0.0029, 0.0025, 0.0024, 0.0024, 0.0022, 0.0027, 0.0018,\n        0.0016, 0.0034, 0.0024, 0.0023, 0.0027, 0.0021, 0.0025, 0.0024, 0.0026,\n        0.0026, 0.0025, 0.0029, 0.0026, 0.0020, 0.0030, 0.0021, 0.0026, 0.0024,\n        0.0022, 0.0035, 0.0028, 0.0026, 0.0028, 0.0026, 0.0026, 0.0031, 0.0024,\n        0.0027, 0.0038, 0.0021, 0.0029, 0.0025, 0.0030, 0.0030, 0.0026, 0.0021,\n        0.0026, 0.0029, 0.0024, 0.0026, 0.0029, 0.0023, 0.0036, 0.0031, 0.0028,\n        0.0024, 0.0025, 0.0023, 0.0031, 0.0027, 0.0024, 0.0022, 0.0025, 0.0031,\n        0.0024, 0.0021, 0.0024, 0.0021, 0.0023, 0.0026, 0.0025, 0.0025, 0.0025,\n        0.0020, 0.0031, 0.0033, 0.0020, 0.0045, 0.0023, 0.0033, 0.0024, 0.0030,\n        0.0022, 0.0022, 0.0023, 0.0020, 0.0026, 0.0031, 0.0016, 0.0027, 0.0025,\n        0.0026, 0.0032, 0.0021, 0.0033, 0.0029, 0.0024, 0.0030, 0.0027, 0.0025,\n        0.0024], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([ 3.7015e-04, -2.3370e-03,  9.0641e-03,  4.4973e-03, -1.0452e-02,\n         4.3782e-03, -2.0823e-03, -7.3727e-03, -1.1328e-02, -1.8645e-02,\n         2.3319e-02, -7.7803e-03, -4.1483e-03, -1.0812e-02, -1.3694e-02,\n        -4.7227e-03, -1.9605e-02,  4.1987e-02, -1.3836e-02,  1.3592e-02,\n        -2.3378e-03, -1.4993e-02, -1.2426e-03,  5.2254e-03, -4.8466e-04,\n         5.3871e-03,  3.2941e-03, -3.5580e-03, -4.2602e-03, -7.9428e-03,\n         1.6526e-02,  2.7316e-03,  1.9632e-02, -1.4506e-02, -7.1378e-03,\n        -8.1808e-03,  9.0706e-04, -5.7208e-03,  8.3405e-03, -1.1962e-02,\n         2.1529e-03,  2.7002e-03, -9.5559e-03,  2.5858e-03,  1.9019e-02,\n         1.6488e-02,  1.7670e-02, -1.1011e-02, -4.5329e-03, -3.7018e-02,\n         3.7496e-03,  2.4826e-02,  4.9099e-03, -1.8235e-02,  1.4793e-02,\n        -2.6602e-02, -1.4454e-02,  1.8537e-02,  4.2614e-04,  8.5051e-03,\n        -4.6100e-04,  7.5454e-03, -8.1947e-03,  6.6934e-03, -5.2298e-03,\n         3.7937e-03,  4.4811e-03,  1.1742e-02, -1.3882e-02, -5.8893e-03,\n        -2.5940e-03, -1.0937e-02, -4.5452e-03, -1.7575e-02, -3.3608e-03,\n        -1.0778e-02,  1.4423e-02,  2.1711e-03, -8.6503e-03, -3.9028e-02,\n        -6.9429e-03,  1.0434e-02, -8.4117e-03, -2.5737e-03, -1.9437e-02,\n         1.2141e-02, -8.9491e-03, -1.5660e-02, -7.8207e-03,  2.7758e-03,\n         2.4170e-03,  1.1981e-02, -1.7890e-02,  1.9754e-03, -1.0491e-02,\n        -1.7669e-03, -7.6529e-03,  1.7853e-02, -6.1149e-03,  2.9477e-04,\n         2.0961e-02, -2.3410e-02,  1.0970e-02,  1.1934e-02,  2.4891e-03,\n         4.3455e-02,  3.5481e-03,  7.4252e-03,  1.1450e-02,  1.5432e-02,\n        -1.8916e-04, -1.9861e-02,  1.5046e-03, -1.1228e-02, -2.7466e-02,\n        -6.5935e-04,  6.0407e-03,  3.5843e-03,  6.4985e-03, -5.6037e-03,\n        -7.3331e-03, -1.8326e-02, -6.3276e-03, -1.0016e-03,  1.8660e-02,\n         1.3386e-02,  6.2620e-03,  7.7024e-03, -9.7012e-03,  1.3455e-03,\n        -2.2261e-03,  1.1020e-03,  5.5393e-03, -4.2593e-03,  1.7430e-02,\n        -2.3933e-02, -2.5678e-02,  1.1107e-02, -2.3952e-03, -2.0671e-02,\n         5.0374e-03,  3.1478e-02, -1.2650e-02,  2.3486e-02, -4.7556e-03,\n         2.5522e-03,  2.1172e-03, -1.3519e-02, -2.7345e-03, -1.8219e-02,\n         3.2386e-03,  2.6787e-03, -3.7405e-02,  2.0377e-03,  2.2786e-02,\n        -3.7128e-05, -3.9497e-02,  5.3251e-03,  1.3003e-02,  6.6501e-03,\n        -3.2034e-02,  3.6701e-02, -2.2223e-04,  1.2996e-02,  1.0601e-02,\n        -1.9193e-03,  8.2985e-03,  9.2626e-04,  1.8440e-03,  7.9549e-03,\n         9.4247e-03,  1.7267e-02, -6.2791e-03,  3.8940e-03,  2.0544e-03,\n         1.3940e-03, -1.1052e-02, -8.3617e-03,  7.2351e-03,  4.1323e-03,\n         6.8777e-04, -2.0998e-02, -1.0373e-03, -4.6615e-03, -1.3763e-02,\n        -1.1012e-03, -2.8058e-02,  3.5221e-03,  2.3242e-03, -3.7730e-02,\n         4.1052e-03,  9.4342e-03,  7.4152e-03,  2.3496e-02, -3.4740e-03,\n         3.7744e-03, -3.1437e-02, -2.9619e-03,  1.0581e-02, -2.4242e-02,\n        -2.8182e-03,  5.5457e-03, -1.0421e-02,  8.6784e-03, -2.8856e-03,\n         5.0659e-04,  9.8441e-03, -3.0326e-04,  3.0359e-03, -9.8104e-03,\n        -9.9844e-03,  1.4239e-02, -2.5225e-02,  1.0159e-03, -1.2108e-02,\n         7.0868e-03, -1.2549e-02,  1.4220e-02,  4.4437e-03,  2.0620e-02,\n        -2.7853e-03,  2.3273e-02,  9.1179e-03, -1.7696e-02,  1.1397e-02,\n         2.5824e-02, -4.4603e-03, -2.1303e-02,  1.3596e-02,  1.6398e-02,\n         8.5609e-03, -4.6444e-04,  1.5405e-02,  1.8042e-02, -1.0500e-02,\n        -6.6665e-02,  1.2047e-02,  2.0606e-02,  2.1219e-02,  1.1406e-02,\n         3.6286e-02, -2.8446e-03, -6.4342e-03, -8.0879e-03,  1.0519e-02,\n         4.8636e-03, -3.1575e-02,  2.8921e-02,  1.4505e-02, -5.5999e-03,\n        -6.3928e-03,  1.8554e-03,  1.4512e-02,  4.0229e-03, -3.3347e-02,\n        -8.0840e-03,  4.7516e-03,  7.2561e-03,  8.3234e-03,  3.6309e-02,\n        -2.2225e-02, -3.1745e-03, -1.3161e-02,  2.3942e-03, -9.7760e-03,\n         4.1599e-03, -5.9595e-03,  8.3747e-03,  7.7639e-03, -7.7427e-03,\n         6.2609e-03, -9.3417e-04, -2.5982e-02,  3.1943e-02, -8.4646e-04,\n         1.3606e-02,  2.8375e-03, -1.2618e-02,  1.3730e-02, -5.0016e-03,\n        -1.3591e-02, -5.3509e-03, -1.8622e-02,  1.0042e-02,  1.2241e-03,\n         2.0266e-02,  8.7766e-03,  5.2801e-03, -1.1249e-02, -9.1190e-03,\n         1.8848e-02,  3.4133e-03,  1.9920e-02,  1.5360e-02, -8.1308e-03,\n         9.7479e-03,  2.5615e-03,  3.2904e-04,  5.3236e-03,  1.1275e-02,\n        -2.3847e-02,  8.8524e-03,  4.7067e-03, -1.2748e-03,  8.6914e-03,\n         1.6651e-02,  1.2849e-02,  9.0641e-03,  4.8100e-03,  1.8057e-02,\n         6.7364e-03, -1.6841e-03, -1.8297e-02, -3.9983e-03, -1.5536e-02,\n         4.5254e-03, -1.3754e-02,  5.4797e-03, -1.3434e-02, -2.5416e-02,\n         3.6965e-02,  1.0310e-02, -1.0124e-04,  1.7364e-03, -1.0163e-02,\n         1.5583e-02, -3.6903e-03,  2.0873e-02,  8.7366e-03, -3.0092e-02,\n        -2.1265e-02,  3.1959e-03, -3.2141e-02, -2.7559e-02,  1.4039e-02,\n         7.3270e-03, -2.5510e-03,  1.8156e-03,  4.2029e-02,  3.1559e-03,\n         3.0051e-02,  4.4247e-03,  6.7175e-03, -1.3601e-02,  2.0102e-03,\n        -4.3502e-03, -4.1042e-02,  2.7482e-03,  3.8990e-03,  6.0767e-03,\n        -1.8549e-03,  4.3223e-03,  1.2431e-03,  2.5267e-03,  2.1141e-02,\n        -5.4905e-03, -1.9051e-02, -2.0751e-03,  2.4948e-03, -1.0516e-02,\n        -9.1798e-04,  4.7818e-03, -2.6325e-03, -1.2754e-02,  9.3676e-03,\n         5.5414e-03,  2.0135e-02, -3.3297e-03, -6.5511e-04, -1.5180e-02,\n        -6.2350e-02, -2.2458e-02, -4.0152e-03,  1.8771e-02, -2.8697e-03,\n         2.4316e-03, -4.5374e-03,  1.2434e-02,  2.1114e-02,  1.2682e-02,\n         3.0717e-02,  8.1329e-03,  1.7098e-02,  1.1113e-02, -9.7007e-03,\n        -3.2172e-03, -9.4604e-03,  1.8473e-03, -1.1092e-02, -9.0469e-04,\n         9.9962e-03, -3.0490e-02,  3.8586e-02, -5.0345e-03, -2.7900e-02,\n         1.4848e-02,  3.8165e-02,  1.9578e-02, -1.4309e-02,  8.1076e-03,\n         4.0688e-03, -1.8150e-02, -1.8551e-02, -6.9978e-03,  7.6080e-03,\n         2.3558e-02, -9.7192e-03, -2.5298e-03, -9.8672e-03,  2.0012e-02,\n         3.3717e-03,  1.5531e-02, -2.3598e-03, -1.4062e-02,  1.8739e-03,\n        -6.2653e-04, -1.0313e-02, -1.6515e-03, -3.5710e-03, -1.7260e-02,\n         5.1800e-04,  1.8365e-02,  2.4575e-02, -9.3078e-03,  1.1834e-02,\n         2.6459e-02,  2.3392e-02, -1.8705e-02, -7.9415e-03, -5.7631e-03,\n         2.8511e-02,  1.7752e-02, -1.0011e-02,  7.0031e-03, -2.5317e-02,\n         1.1355e-02,  2.3412e-02, -7.0986e-03, -5.8811e-03,  1.9290e-03,\n        -1.0360e-02, -6.3055e-03, -9.6267e-03,  1.4141e-02,  1.8005e-03,\n        -1.4323e-02,  1.3786e-02,  1.5624e-02,  1.7800e-02, -1.3349e-02,\n        -8.3177e-03,  2.5931e-02, -3.0994e-04,  1.6163e-02,  8.3758e-04,\n         1.1851e-02,  1.8527e-02,  2.5990e-02,  1.1613e-02, -3.6058e-02,\n         3.0730e-03, -1.3725e-02,  2.1578e-03, -1.0197e-02, -2.6884e-02,\n         3.8387e-03, -2.2237e-02,  1.1961e-02, -5.8919e-03,  2.2816e-02,\n         2.4111e-04, -2.4864e-02, -1.2242e-02, -1.0923e-02, -2.8317e-03,\n         5.0746e-04,  1.7315e-02,  2.0425e-03,  6.3374e-03, -6.5011e-03,\n         4.1144e-03, -2.4692e-02,  2.1480e-02,  4.0464e-06,  1.4859e-02,\n        -1.8319e-02,  8.1660e-03, -3.7029e-03,  1.8128e-03, -2.7395e-02,\n         1.5815e-02, -4.2541e-03,  2.0555e-03, -1.3953e-02,  7.0785e-04,\n         1.4946e-02,  2.7037e-03,  1.4192e-02,  1.0791e-02,  1.2503e-02,\n         5.7559e-04, -7.3741e-03, -1.0589e-02,  1.2760e-02,  5.7868e-03,\n         7.7232e-03, -1.2102e-02,  1.5461e-02, -2.9322e-02, -3.7516e-02,\n         1.9406e-02,  3.7544e-03, -4.9696e-03,  1.7453e-02,  1.9470e-02,\n         1.9318e-02,  1.2881e-03,  2.6887e-02, -2.3501e-03, -7.0118e-03,\n        -1.3360e-02,  1.2180e-03,  4.9717e-03,  3.3087e-02, -1.6251e-02,\n        -1.9534e-02,  7.5200e-03, -1.0589e-02,  6.5200e-04,  1.7015e-02,\n         8.2721e-03,  1.4412e-02, -1.6038e-03,  7.1778e-03,  5.6243e-04,\n         3.3249e-03, -3.6018e-03,  1.9255e-02, -1.3330e-03, -5.6798e-03,\n        -1.0747e-02, -1.0796e-02, -1.4548e-02,  1.9716e-03, -8.7362e-02,\n         1.7665e-02,  1.8212e-02, -4.2831e-03,  8.6517e-03,  2.0562e-02,\n         5.7245e-03,  1.3288e-04, -3.1388e-02, -7.4862e-03,  1.2812e-05,\n         5.4817e-03,  1.6561e-02, -3.5099e-05,  1.4302e-02,  5.5100e-03,\n         8.0109e-03, -2.0828e-02,  2.6218e-03,  5.5258e-03,  8.2771e-03,\n         4.3478e-03,  1.7280e-02,  3.8516e-05,  5.3053e-04,  2.4896e-02,\n        -2.4664e-02, -7.6390e-03,  2.5688e-03, -2.9710e-03, -4.1264e-03,\n        -1.6960e-03,  2.2620e-03,  2.5374e-02,  1.3910e-02, -8.4825e-04,\n         1.4985e-03,  1.0969e-04,  4.4948e-03,  1.8173e-02, -2.7873e-02,\n        -1.3119e-02,  2.2612e-02,  1.1584e-02, -1.7831e-03,  1.9717e-02,\n        -2.0277e-04,  1.6129e-02,  1.1356e-02, -8.4536e-03,  1.0369e-02,\n        -2.4749e-02, -6.5893e-03,  4.1463e-04, -8.3810e-04,  1.1518e-02,\n        -2.2795e-02, -3.3748e-02,  9.6555e-03,  1.1948e-02, -1.4417e-02,\n         7.2138e-03, -4.5353e-03, -4.5611e-04,  8.8891e-03,  4.3160e-03,\n         3.8857e-03, -3.4371e-02, -7.1428e-03, -8.5142e-03, -4.9979e-03,\n         3.3383e-02, -2.1196e-02, -1.6020e-03,  1.4625e-02,  8.6971e-03,\n        -1.3028e-02, -6.6353e-03,  1.2731e-02,  5.4896e-03, -1.0384e-02,\n         8.6397e-03, -4.2818e-04, -8.7563e-03,  9.3405e-03,  5.3446e-03,\n         9.2650e-03, -1.1889e-02,  3.1683e-03, -1.6852e-02, -1.5115e-02,\n        -1.8536e-02, -7.4602e-03, -1.6525e-03, -2.8242e-02, -1.4293e-02],\n       device='cuda:0')",
+        "exp_avg_sq": "tensor([0.0019, 0.0019, 0.0018, 0.0019, 0.0023, 0.0023, 0.0012, 0.0018, 0.0022,\n        0.0023, 0.0031, 0.0029, 0.0029, 0.0029, 0.0016, 0.0019, 0.0020, 0.0029,\n        0.0016, 0.0025, 0.0023, 0.0020, 0.0020, 0.0027, 0.0020, 0.0029, 0.0042,\n        0.0022, 0.0027, 0.0025, 0.0019, 0.0022, 0.0032, 0.0025, 0.0022, 0.0019,\n        0.0018, 0.0021, 0.0029, 0.0020, 0.0022, 0.0014, 0.0024, 0.0021, 0.0020,\n        0.0026, 0.0025, 0.0022, 0.0022, 0.0019, 0.0021, 0.0019, 0.0020, 0.0024,\n        0.0024, 0.0026, 0.0040, 0.0021, 0.0027, 0.0020, 0.0021, 0.0025, 0.0023,\n        0.0031, 0.0029, 0.0021, 0.0033, 0.0028, 0.0025, 0.0024, 0.0031, 0.0018,\n        0.0031, 0.0029, 0.0020, 0.0022, 0.0020, 0.0017, 0.0019, 0.0023, 0.0020,\n        0.0020, 0.0018, 0.0025, 0.0018, 0.0019, 0.0018, 0.0026, 0.0023, 0.0021,\n        0.0023, 0.0034, 0.0023, 0.0017, 0.0026, 0.0021, 0.0019, 0.0020, 0.0018,\n        0.0024, 0.0024, 0.0017, 0.0021, 0.0018, 0.0021, 0.0028, 0.0019, 0.0021,\n        0.0034, 0.0023, 0.0025, 0.0023, 0.0019, 0.0023, 0.0021, 0.0020, 0.0024,\n        0.0023, 0.0020, 0.0036, 0.0027, 0.0030, 0.0019, 0.0020, 0.0025, 0.0025,\n        0.0040, 0.0021, 0.0021, 0.0020, 0.0023, 0.0035, 0.0026, 0.0022, 0.0022,\n        0.0021, 0.0020, 0.0021, 0.0027, 0.0032, 0.0022, 0.0024, 0.0027, 0.0025,\n        0.0025, 0.0021, 0.0017, 0.0023, 0.0023, 0.0017, 0.0022, 0.0020, 0.0033,\n        0.0034, 0.0025, 0.0026, 0.0029, 0.0019, 0.0023, 0.0025, 0.0025, 0.0034,\n        0.0027, 0.0027, 0.0024, 0.0023, 0.0017, 0.0027, 0.0028, 0.0029, 0.0015,\n        0.0017, 0.0019, 0.0020, 0.0028, 0.0029, 0.0025, 0.0019, 0.0017, 0.0021,\n        0.0031, 0.0024, 0.0023, 0.0019, 0.0018, 0.0021, 0.0025, 0.0021, 0.0029,\n        0.0032, 0.0022, 0.0028, 0.0021, 0.0025, 0.0019, 0.0020, 0.0019, 0.0027,\n        0.0022, 0.0025, 0.0022, 0.0025, 0.0033, 0.0037, 0.0024, 0.0021, 0.0027,\n        0.0022, 0.0023, 0.0024, 0.0023, 0.0016, 0.0022, 0.0034, 0.0022, 0.0035,\n        0.0021, 0.0023, 0.0021, 0.0028, 0.0023, 0.0025, 0.0017, 0.0025, 0.0020,\n        0.0020, 0.0030, 0.0021, 0.0027, 0.0023, 0.0024, 0.0023, 0.0026, 0.0020,\n        0.0024, 0.0031, 0.0021, 0.0028, 0.0021, 0.0027, 0.0022, 0.0022, 0.0021,\n        0.0029, 0.0028, 0.0023, 0.0030, 0.0027, 0.0023, 0.0025, 0.0021, 0.0019,\n        0.0036, 0.0028, 0.0021, 0.0022, 0.0025, 0.0020, 0.0018, 0.0022, 0.0019,\n        0.0023, 0.0017, 0.0032, 0.0029, 0.0023, 0.0019, 0.0016, 0.0021, 0.0018,\n        0.0024, 0.0042, 0.0022, 0.0021, 0.0019, 0.0022, 0.0020, 0.0019, 0.0023,\n        0.0020, 0.0023, 0.0029, 0.0033, 0.0027, 0.0021, 0.0024, 0.0023, 0.0020,\n        0.0031, 0.0024, 0.0020, 0.0018, 0.0019, 0.0017, 0.0022, 0.0027, 0.0023,\n        0.0019, 0.0034, 0.0025, 0.0022, 0.0022, 0.0028, 0.0018, 0.0021, 0.0026,\n        0.0021, 0.0023, 0.0020, 0.0023, 0.0021, 0.0022, 0.0015, 0.0030, 0.0035,\n        0.0025, 0.0025, 0.0024, 0.0021, 0.0021, 0.0023, 0.0031, 0.0023, 0.0022,\n        0.0026, 0.0025, 0.0027, 0.0025, 0.0029, 0.0020, 0.0039, 0.0020, 0.0030,\n        0.0022, 0.0036, 0.0030, 0.0021, 0.0020, 0.0034, 0.0024, 0.0025, 0.0023,\n        0.0019, 0.0024, 0.0029, 0.0027, 0.0020, 0.0021, 0.0024, 0.0026, 0.0018,\n        0.0023, 0.0040, 0.0026, 0.0028, 0.0024, 0.0026, 0.0023, 0.0020, 0.0024,\n        0.0017, 0.0019, 0.0019, 0.0024, 0.0031, 0.0028, 0.0028, 0.0032, 0.0021,\n        0.0029, 0.0028, 0.0032, 0.0021, 0.0028, 0.0029, 0.0030, 0.0029, 0.0028,\n        0.0020, 0.0023, 0.0033, 0.0023, 0.0022, 0.0021, 0.0023, 0.0021, 0.0029,\n        0.0025, 0.0022, 0.0023, 0.0021, 0.0028, 0.0024, 0.0028, 0.0022, 0.0022,\n        0.0021, 0.0025, 0.0024, 0.0041, 0.0020, 0.0025, 0.0024, 0.0024, 0.0023,\n        0.0028, 0.0037, 0.0020, 0.0023, 0.0022, 0.0039, 0.0027, 0.0032, 0.0021,\n        0.0023, 0.0018, 0.0020, 0.0018, 0.0020, 0.0019, 0.0032, 0.0022, 0.0026,\n        0.0029, 0.0019, 0.0022, 0.0027, 0.0027, 0.0021, 0.0035, 0.0022, 0.0024,\n        0.0014, 0.0024, 0.0027, 0.0022, 0.0027, 0.0024, 0.0028, 0.0032, 0.0019,\n        0.0019, 0.0018, 0.0031, 0.0033, 0.0025, 0.0027, 0.0026, 0.0020, 0.0030,\n        0.0038, 0.0019, 0.0021, 0.0027, 0.0026, 0.0020, 0.0025, 0.0029, 0.0028,\n        0.0020, 0.0022, 0.0020, 0.0022, 0.0021, 0.0026, 0.0020, 0.0025, 0.0027,\n        0.0018, 0.0024, 0.0024, 0.0028, 0.0016, 0.0035, 0.0020, 0.0029, 0.0027,\n        0.0017, 0.0021, 0.0022, 0.0036, 0.0017, 0.0026, 0.0026, 0.0024, 0.0024,\n        0.0023, 0.0022, 0.0021, 0.0021, 0.0019, 0.0035, 0.0017, 0.0021, 0.0022,\n        0.0020, 0.0024, 0.0019, 0.0033, 0.0020, 0.0020, 0.0021, 0.0019, 0.0024,\n        0.0022, 0.0025, 0.0026, 0.0025, 0.0023, 0.0028, 0.0023, 0.0027, 0.0016,\n        0.0026, 0.0015, 0.0028, 0.0019, 0.0023, 0.0020, 0.0024, 0.0029, 0.0020,\n        0.0021, 0.0019, 0.0026, 0.0024, 0.0018, 0.0021, 0.0024, 0.0025, 0.0024,\n        0.0018, 0.0016, 0.0020, 0.0028, 0.0022, 0.0025, 0.0023, 0.0018, 0.0026,\n        0.0020, 0.0023, 0.0022, 0.0022, 0.0034, 0.0025, 0.0020, 0.0025, 0.0017,\n        0.0015, 0.0026, 0.0024, 0.0023, 0.0023, 0.0020, 0.0023, 0.0024, 0.0025,\n        0.0024, 0.0022, 0.0026, 0.0024, 0.0018, 0.0027, 0.0019, 0.0021, 0.0020,\n        0.0017, 0.0035, 0.0027, 0.0022, 0.0027, 0.0024, 0.0021, 0.0029, 0.0024,\n        0.0022, 0.0038, 0.0018, 0.0031, 0.0025, 0.0032, 0.0029, 0.0024, 0.0020,\n        0.0024, 0.0029, 0.0019, 0.0020, 0.0029, 0.0020, 0.0036, 0.0029, 0.0024,\n        0.0020, 0.0023, 0.0019, 0.0024, 0.0024, 0.0021, 0.0018, 0.0023, 0.0031,\n        0.0024, 0.0019, 0.0021, 0.0023, 0.0021, 0.0025, 0.0023, 0.0023, 0.0026,\n        0.0018, 0.0029, 0.0031, 0.0019, 0.0050, 0.0022, 0.0032, 0.0021, 0.0031,\n        0.0022, 0.0020, 0.0024, 0.0019, 0.0024, 0.0027, 0.0013, 0.0024, 0.0025,\n        0.0024, 0.0034, 0.0020, 0.0028, 0.0028, 0.0022, 0.0027, 0.0027, 0.0021,\n        0.0022], device='cuda:0')"
       },
       "2": {
-        "step": "tensor(1252.)",
-        "exp_avg": "tensor([[-2.3712e-04,  2.9616e-04,  1.7065e-05,  ..., -4.0576e-04,\n          7.7187e-04,  6.5222e-05],\n        [ 1.9820e-04,  4.1821e-04, -3.7054e-04,  ..., -3.7018e-05,\n         -1.0355e-03,  6.7209e-04],\n        [ 3.2962e-04,  1.8478e-04, -9.8128e-05,  ...,  2.5392e-04,\n         -8.7285e-05, -2.0284e-04],\n        ...,\n        [-5.6400e-04,  6.4011e-05, -3.3537e-04,  ..., -1.0095e-04,\n         -7.4733e-04,  3.5475e-04],\n        [ 9.9958e-05, -1.3464e-04, -6.6557e-05,  ..., -1.0789e-04,\n         -1.3525e-04,  3.5616e-04],\n        [-3.1181e-04, -1.2565e-04, -2.9232e-04,  ...,  3.3805e-04,\n         -2.1624e-04,  5.8790e-04]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[1.4221e-06, 7.4916e-07, 1.3669e-06,  ..., 6.6120e-07, 1.6104e-06,\n         1.5282e-06],\n        [1.8296e-06, 1.8416e-06, 2.2270e-06,  ..., 1.7167e-06, 3.0106e-06,\n         2.4378e-06],\n        [1.8387e-06, 2.3293e-06, 2.3369e-06,  ..., 1.6733e-06, 2.0745e-06,\n         1.6583e-06],\n        ...,\n        [2.3226e-06, 1.7147e-06, 2.8430e-06,  ..., 1.6356e-06, 2.7633e-06,\n         4.0792e-06],\n        [2.1010e-06, 2.3743e-06, 2.7163e-06,  ..., 1.5971e-06, 2.6313e-06,\n         2.0676e-06],\n        [2.5352e-06, 1.1932e-06, 3.0873e-06,  ..., 1.8835e-06, 2.8250e-06,\n         2.4812e-06]], device='cuda:0')"
+        "step": "tensor(2504.)",
+        "exp_avg": "tensor([[ 3.3157e-05,  9.0172e-05,  8.2628e-05,  ...,  5.1896e-05,\n          9.1822e-05, -2.2610e-04],\n        [-4.5172e-04, -1.2079e-04,  1.8322e-04,  ..., -3.3412e-05,\n          1.8097e-04,  1.7297e-04],\n        [ 4.6973e-04, -8.3150e-05, -9.6866e-04,  ..., -3.8514e-05,\n          1.4621e-04, -6.8942e-05],\n        ...,\n        [-5.8888e-04, -3.6048e-04, -3.6673e-05,  ..., -1.5665e-05,\n         -4.0896e-04,  5.4799e-04],\n        [-1.2193e-03, -3.9815e-04,  3.8778e-04,  ..., -5.7737e-05,\n         -7.0575e-05,  2.8701e-05],\n        [ 1.0101e-04,  1.1042e-04,  2.3041e-04,  ...,  1.5737e-05,\n         -3.5228e-04, -4.9031e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[8.4538e-07, 6.2770e-07, 8.0637e-07,  ..., 3.5103e-07, 8.9550e-07,\n         1.4680e-06],\n        [1.2847e-06, 1.6966e-06, 1.4330e-06,  ..., 9.8359e-07, 2.4769e-06,\n         2.1182e-06],\n        [1.5711e-06, 1.8740e-06, 1.7353e-06,  ..., 1.0199e-06, 1.4728e-06,\n         1.2872e-06],\n        ...,\n        [1.6384e-06, 1.4097e-06, 1.7026e-06,  ..., 1.1115e-06, 1.9571e-06,\n         3.8118e-06],\n        [1.8098e-06, 2.3717e-06, 1.8668e-06,  ..., 8.7579e-07, 1.8769e-06,\n         1.9211e-06],\n        [2.0767e-06, 8.3075e-07, 2.4246e-06,  ..., 1.2058e-06, 1.8157e-06,\n         2.2155e-06]], device='cuda:0')"
       }
     },
     "param_groups": [
       {
-        "lr": 0.0009755527298894294,
+        "lr": 0.0009046039886902864,
         "name": "scale_256",
         "betas": [
           0.9,
@@ -43,7 +43,7 @@
         ]
       },
       {
-        "lr": 0.0009755527298894294,
+        "lr": 0.0009046039886902864,
         "name": "scale_512",
         "betas": [
           0.9,
@@ -66,7 +66,7 @@
         ]
       },
       {
-        "lr": 0.0009755527298894294,
+        "lr": 0.0009046039886902864,
         "name": "scale_768",
         "betas": [
           0.9,
@@ -89,7 +89,7 @@
         ]
       },
       {
-        "lr": 0.0009755527298894294,
+        "lr": 0.0009046039886902864,
         "name": "scale_1024",
         "betas": [
           0.9,
@@ -112,7 +112,7 @@
         ]
       },
       {
-        "lr": 0.0009755527298894294,
+        "lr": 0.0009046039886902864,
         "name": "scale_1280",
         "betas": [
           0.9,
@@ -135,7 +135,7 @@
         ]
       },
       {
-        "lr": 0.00048778860081564085,
+        "lr": 0.0004523497400965494,
         "name": "fusion",
         "betas": [
           0.9,
@@ -195,7 +195,7 @@
     "T_i": 10,
     "T_mult": 2,
     "eta_min": 1e-06,
-    "T_cur": 1,
+    "T_cur": 2,
     "base_lrs": [
       0.001,
       0.001,
@@ -204,24 +204,24 @@
       0.001,
       0.0005
     ],
-    "last_epoch": 1,
+    "last_epoch": 2,
     "_step_count": 0,
     "_is_initial": false,
     "_get_lr_called_within_step": false,
     "_last_lr": [
-      0.0009755527298894294,
-      0.0009755527298894294,
-      0.0009755527298894294,
-      0.0009755527298894294,
-      0.0009755527298894294,
-      0.00048778860081564085
+      0.0009046039886902864,
+      0.0009046039886902864,
+      0.0009046039886902864,
+      0.0009046039886902864,
+      0.0009046039886902864,
+      0.0004523497400965494
     ]
   },
   "metrics": {
-    "best_val_acc": 69.492,
-    "best_epoch": 0,
+    "best_val_acc": 71.226,
+    "best_epoch": 1,
     "scale_accuracies": {
-      "256": 69.492
+      "256": 71.226
     }
   },
   "train_config": {