Upload weights and configs - David-decoupled-deep_efficiency - Run 20251012_065325

weights/David-decoupled-deep_efficiency/20251012_065325/best_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f37bf4247645adf0df795b72b8f62941acd742446a82d070425620418d2292d
+oid sha256:63da40ca37b1d6dfef6fe878c9795165ecd148a58435169b369b095f7bdf5908
 size 59515088

weights/David-decoupled-deep_efficiency/20251012_065325/best_model_metadata.json

@@ -1,26 +1,26 @@
 {
-  "epoch": 1,
+  "epoch": 2,
   "optimizer_state_dict": {
     "state": {
       "0": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([[ 1.9903e-04, -2.0316e-03,  4.0099e-04,  ...,  3.9600e-04,\n         -7.7503e-05, -3.2745e-04],\n        [ 1.2035e-04, -5.4277e-04,  8.3932e-04,  ..., -6.4219e-05,\n         -1.1444e-04,  2.8184e-04],\n        [-6.7256e-05,  9.3359e-04, -1.2770e-04,  ...,  2.4264e-04,\n         -4.6260e-04, -2.6548e-04],\n        ...,\n        [-9.3872e-06, -3.3246e-04, -1.0106e-04,  ..., -6.8116e-05,\n         -1.0831e-05, -4.9642e-04],\n        [-1.2853e-03, -3.1639e-03,  1.6793e-03,  ...,  8.5221e-04,\n          2.0776e-04, -5.3517e-04],\n        [-6.0768e-04, -1.0425e-03,  2.5003e-04,  ...,  1.3259e-04,\n         -6.0654e-05, -1.6917e-04]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[1.4933e-06, 1.3383e-05, 4.9550e-06,  ..., 2.0071e-06, 1.0758e-06,\n         1.6587e-06],\n        [3.3611e-06, 9.2764e-06, 9.0210e-06,  ..., 2.5916e-06, 9.1690e-07,\n         1.3613e-06],\n        [2.5602e-06, 1.6735e-05, 6.9803e-06,  ..., 1.5625e-06, 1.0142e-06,\n         1.3210e-06],\n        ...,\n        [4.2617e-06, 3.2869e-05, 9.4941e-06,  ..., 2.8932e-06, 1.9778e-06,\n         1.9807e-06],\n        [2.4394e-06, 2.4098e-05, 7.0105e-06,  ..., 1.8268e-06, 1.1096e-06,\n         1.5990e-06],\n        [2.7173e-06, 1.4306e-05, 5.9305e-06,  ..., 1.5879e-06, 1.2643e-06,\n         3.5216e-06]], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[ 6.2482e-05, -1.0957e-03,  1.8610e-04,  ..., -6.3775e-05,\n          3.7722e-05,  3.0946e-04],\n        [ 4.2669e-04, -4.4703e-04,  4.8625e-05,  ..., -5.4459e-04,\n         -1.1069e-04, -1.9971e-04],\n        [ 2.1516e-04,  4.9376e-04,  1.0133e-03,  ..., -5.4277e-04,\n         -5.4259e-04, -2.4519e-04],\n        ...,\n        [ 1.3256e-04,  4.0228e-04, -8.2632e-04,  ...,  2.7256e-05,\n         -3.2793e-04,  1.5244e-04],\n        [-4.2531e-04, -3.0743e-03,  1.4001e-04,  ..., -4.0860e-04,\n          4.6036e-04, -1.8925e-04],\n        [-1.4353e-04, -1.3795e-04, -6.5146e-04,  ..., -1.1998e-04,\n          7.3662e-05, -3.9551e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[1.3045e-06, 1.2095e-05, 3.8998e-06,  ..., 1.8005e-06, 9.7567e-07,\n         1.2811e-06],\n        [3.1867e-06, 7.4417e-06, 8.1910e-06,  ..., 2.5037e-06, 8.5574e-07,\n         1.1760e-06],\n        [2.4247e-06, 1.5653e-05, 6.0696e-06,  ..., 1.4071e-06, 9.6109e-07,\n         1.2029e-06],\n        ...,\n        [4.4321e-06, 3.4483e-05, 8.9087e-06,  ..., 2.7869e-06, 1.6946e-06,\n         1.8854e-06],\n        [1.9278e-06, 2.0316e-05, 5.4190e-06,  ..., 1.3752e-06, 8.8687e-07,\n         1.3061e-06],\n        [2.3611e-06, 1.1804e-05, 4.7666e-06,  ..., 1.2768e-06, 1.1256e-06,\n         2.6667e-06]], device='cuda:0')"
       },
       "1": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([ 3.7015e-04, -2.3370e-03,  9.0641e-03,  4.4973e-03, -1.0452e-02,\n         4.3782e-03, -2.0823e-03, -7.3727e-03, -1.1328e-02, -1.8645e-02,\n         2.3319e-02, -7.7803e-03, -4.1483e-03, -1.0812e-02, -1.3694e-02,\n        -4.7227e-03, -1.9605e-02,  4.1987e-02, -1.3836e-02,  1.3592e-02,\n        -2.3378e-03, -1.4993e-02, -1.2426e-03,  5.2254e-03, -4.8466e-04,\n         5.3871e-03,  3.2941e-03, -3.5580e-03, -4.2602e-03, -7.9428e-03,\n         1.6526e-02,  2.7316e-03,  1.9632e-02, -1.4506e-02, -7.1378e-03,\n        -8.1808e-03,  9.0706e-04, -5.7208e-03,  8.3405e-03, -1.1962e-02,\n         2.1529e-03,  2.7002e-03, -9.5559e-03,  2.5858e-03,  1.9019e-02,\n         1.6488e-02,  1.7670e-02, -1.1011e-02, -4.5329e-03, -3.7018e-02,\n         3.7496e-03,  2.4826e-02,  4.9099e-03, -1.8235e-02,  1.4793e-02,\n        -2.6602e-02, -1.4454e-02,  1.8537e-02,  4.2614e-04,  8.5051e-03,\n        -4.6100e-04,  7.5454e-03, -8.1947e-03,  6.6934e-03, -5.2298e-03,\n         3.7937e-03,  4.4811e-03,  1.1742e-02, -1.3882e-02, -5.8893e-03,\n        -2.5940e-03, -1.0937e-02, -4.5452e-03, -1.7575e-02, -3.3608e-03,\n        -1.0778e-02,  1.4423e-02,  2.1711e-03, -8.6503e-03, -3.9028e-02,\n        -6.9429e-03,  1.0434e-02, -8.4117e-03, -2.5737e-03, -1.9437e-02,\n         1.2141e-02, -8.9491e-03, -1.5660e-02, -7.8207e-03,  2.7758e-03,\n         2.4170e-03,  1.1981e-02, -1.7890e-02,  1.9754e-03, -1.0491e-02,\n        -1.7669e-03, -7.6529e-03,  1.7853e-02, -6.1149e-03,  2.9477e-04,\n         2.0961e-02, -2.3410e-02,  1.0970e-02,  1.1934e-02,  2.4891e-03,\n         4.3455e-02,  3.5481e-03,  7.4252e-03,  1.1450e-02,  1.5432e-02,\n        -1.8916e-04, -1.9861e-02,  1.5046e-03, -1.1228e-02, -2.7466e-02,\n        -6.5935e-04,  6.0407e-03,  3.5843e-03,  6.4985e-03, -5.6037e-03,\n        -7.3331e-03, -1.8326e-02, -6.3276e-03, -1.0016e-03,  1.8660e-02,\n         1.3386e-02,  6.2620e-03,  7.7024e-03, -9.7012e-03,  1.3455e-03,\n        -2.2261e-03,  1.1020e-03,  5.5393e-03, -4.2593e-03,  1.7430e-02,\n        -2.3933e-02, -2.5678e-02,  1.1107e-02, -2.3952e-03, -2.0671e-02,\n         5.0374e-03,  3.1478e-02, -1.2650e-02,  2.3486e-02, -4.7556e-03,\n         2.5522e-03,  2.1172e-03, -1.3519e-02, -2.7345e-03, -1.8219e-02,\n         3.2386e-03,  2.6787e-03, -3.7405e-02,  2.0377e-03,  2.2786e-02,\n        -3.7128e-05, -3.9497e-02,  5.3251e-03,  1.3003e-02,  6.6501e-03,\n        -3.2034e-02,  3.6701e-02, -2.2223e-04,  1.2996e-02,  1.0601e-02,\n        -1.9193e-03,  8.2985e-03,  9.2626e-04,  1.8440e-03,  7.9549e-03,\n         9.4247e-03,  1.7267e-02, -6.2791e-03,  3.8940e-03,  2.0544e-03,\n         1.3940e-03, -1.1052e-02, -8.3617e-03,  7.2351e-03,  4.1323e-03,\n         6.8777e-04, -2.0998e-02, -1.0373e-03, -4.6615e-03, -1.3763e-02,\n        -1.1012e-03, -2.8058e-02,  3.5221e-03,  2.3242e-03, -3.7730e-02,\n         4.1052e-03,  9.4342e-03,  7.4152e-03,  2.3496e-02, -3.4740e-03,\n         3.7744e-03, -3.1437e-02, -2.9619e-03,  1.0581e-02, -2.4242e-02,\n        -2.8182e-03,  5.5457e-03, -1.0421e-02,  8.6784e-03, -2.8856e-03,\n         5.0659e-04,  9.8441e-03, -3.0326e-04,  3.0359e-03, -9.8104e-03,\n        -9.9844e-03,  1.4239e-02, -2.5225e-02,  1.0159e-03, -1.2108e-02,\n         7.0868e-03, -1.2549e-02,  1.4220e-02,  4.4437e-03,  2.0620e-02,\n        -2.7853e-03,  2.3273e-02,  9.1179e-03, -1.7696e-02,  1.1397e-02,\n         2.5824e-02, -4.4603e-03, -2.1303e-02,  1.3596e-02,  1.6398e-02,\n         8.5609e-03, -4.6444e-04,  1.5405e-02,  1.8042e-02, -1.0500e-02,\n        -6.6665e-02,  1.2047e-02,  2.0606e-02,  2.1219e-02,  1.1406e-02,\n         3.6286e-02, -2.8446e-03, -6.4342e-03, -8.0879e-03,  1.0519e-02,\n         4.8636e-03, -3.1575e-02,  2.8921e-02,  1.4505e-02, -5.5999e-03,\n        -6.3928e-03,  1.8554e-03,  1.4512e-02,  4.0229e-03, -3.3347e-02,\n        -8.0840e-03,  4.7516e-03,  7.2561e-03,  8.3234e-03,  3.6309e-02,\n        -2.2225e-02, -3.1745e-03, -1.3161e-02,  2.3942e-03, -9.7760e-03,\n         4.1599e-03, -5.9595e-03,  8.3747e-03,  7.7639e-03, -7.7427e-03,\n         6.2609e-03, -9.3417e-04, -2.5982e-02,  3.1943e-02, -8.4646e-04,\n         1.3606e-02,  2.8375e-03, -1.2618e-02,  1.3730e-02, -5.0016e-03,\n        -1.3591e-02, -5.3509e-03, -1.8622e-02,  1.0042e-02,  1.2241e-03,\n         2.0266e-02,  8.7766e-03,  5.2801e-03, -1.1249e-02, -9.1190e-03,\n         1.8848e-02,  3.4133e-03,  1.9920e-02,  1.5360e-02, -8.1308e-03,\n         9.7479e-03,  2.5615e-03,  3.2904e-04,  5.3236e-03,  1.1275e-02,\n        -2.3847e-02,  8.8524e-03,  4.7067e-03, -1.2748e-03,  8.6914e-03,\n         1.6651e-02,  1.2849e-02,  9.0641e-03,  4.8100e-03,  1.8057e-02,\n         6.7364e-03, -1.6841e-03, -1.8297e-02, -3.9983e-03, -1.5536e-02,\n         4.5254e-03, -1.3754e-02,  5.4797e-03, -1.3434e-02, -2.5416e-02,\n         3.6965e-02,  1.0310e-02, -1.0124e-04,  1.7364e-03, -1.0163e-02,\n         1.5583e-02, -3.6903e-03,  2.0873e-02,  8.7366e-03, -3.0092e-02,\n        -2.1265e-02,  3.1959e-03, -3.2141e-02, -2.7559e-02,  1.4039e-02,\n         7.3270e-03, -2.5510e-03,  1.8156e-03,  4.2029e-02,  3.1559e-03,\n         3.0051e-02,  4.4247e-03,  6.7175e-03, -1.3601e-02,  2.0102e-03,\n        -4.3502e-03, -4.1042e-02,  2.7482e-03,  3.8990e-03,  6.0767e-03,\n        -1.8549e-03,  4.3223e-03,  1.2431e-03,  2.5267e-03,  2.1141e-02,\n        -5.4905e-03, -1.9051e-02, -2.0751e-03,  2.4948e-03, -1.0516e-02,\n        -9.1798e-04,  4.7818e-03, -2.6325e-03, -1.2754e-02,  9.3676e-03,\n         5.5414e-03,  2.0135e-02, -3.3297e-03, -6.5511e-04, -1.5180e-02,\n        -6.2350e-02, -2.2458e-02, -4.0152e-03,  1.8771e-02, -2.8697e-03,\n         2.4316e-03, -4.5374e-03,  1.2434e-02,  2.1114e-02,  1.2682e-02,\n         3.0717e-02,  8.1329e-03,  1.7098e-02,  1.1113e-02, -9.7007e-03,\n        -3.2172e-03, -9.4604e-03,  1.8473e-03, -1.1092e-02, -9.0469e-04,\n         9.9962e-03, -3.0490e-02,  3.8586e-02, -5.0345e-03, -2.7900e-02,\n         1.4848e-02,  3.8165e-02,  1.9578e-02, -1.4309e-02,  8.1076e-03,\n         4.0688e-03, -1.8150e-02, -1.8551e-02, -6.9978e-03,  7.6080e-03,\n         2.3558e-02, -9.7192e-03, -2.5298e-03, -9.8672e-03,  2.0012e-02,\n         3.3717e-03,  1.5531e-02, -2.3598e-03, -1.4062e-02,  1.8739e-03,\n        -6.2653e-04, -1.0313e-02, -1.6515e-03, -3.5710e-03, -1.7260e-02,\n         5.1800e-04,  1.8365e-02,  2.4575e-02, -9.3078e-03,  1.1834e-02,\n         2.6459e-02,  2.3392e-02, -1.8705e-02, -7.9415e-03, -5.7631e-03,\n         2.8511e-02,  1.7752e-02, -1.0011e-02,  7.0031e-03, -2.5317e-02,\n         1.1355e-02,  2.3412e-02, -7.0986e-03, -5.8811e-03,  1.9290e-03,\n        -1.0360e-02, -6.3055e-03, -9.6267e-03,  1.4141e-02,  1.8005e-03,\n        -1.4323e-02,  1.3786e-02,  1.5624e-02,  1.7800e-02, -1.3349e-02,\n        -8.3177e-03,  2.5931e-02, -3.0994e-04,  1.6163e-02,  8.3758e-04,\n         1.1851e-02,  1.8527e-02,  2.5990e-02,  1.1613e-02, -3.6058e-02,\n         3.0730e-03, -1.3725e-02,  2.1578e-03, -1.0197e-02, -2.6884e-02,\n         3.8387e-03, -2.2237e-02,  1.1961e-02, -5.8919e-03,  2.2816e-02,\n         2.4111e-04, -2.4864e-02, -1.2242e-02, -1.0923e-02, -2.8317e-03,\n         5.0746e-04,  1.7315e-02,  2.0425e-03,  6.3374e-03, -6.5011e-03,\n         4.1144e-03, -2.4692e-02,  2.1480e-02,  4.0464e-06,  1.4859e-02,\n        -1.8319e-02,  8.1660e-03, -3.7029e-03,  1.8128e-03, -2.7395e-02,\n         1.5815e-02, -4.2541e-03,  2.0555e-03, -1.3953e-02,  7.0785e-04,\n         1.4946e-02,  2.7037e-03,  1.4192e-02,  1.0791e-02,  1.2503e-02,\n         5.7559e-04, -7.3741e-03, -1.0589e-02,  1.2760e-02,  5.7868e-03,\n         7.7232e-03, -1.2102e-02,  1.5461e-02, -2.9322e-02, -3.7516e-02,\n         1.9406e-02,  3.7544e-03, -4.9696e-03,  1.7453e-02,  1.9470e-02,\n         1.9318e-02,  1.2881e-03,  2.6887e-02, -2.3501e-03, -7.0118e-03,\n        -1.3360e-02,  1.2180e-03,  4.9717e-03,  3.3087e-02, -1.6251e-02,\n        -1.9534e-02,  7.5200e-03, -1.0589e-02,  6.5200e-04,  1.7015e-02,\n         8.2721e-03,  1.4412e-02, -1.6038e-03,  7.1778e-03,  5.6243e-04,\n         3.3249e-03, -3.6018e-03,  1.9255e-02, -1.3330e-03, -5.6798e-03,\n        -1.0747e-02, -1.0796e-02, -1.4548e-02,  1.9716e-03, -8.7362e-02,\n         1.7665e-02,  1.8212e-02, -4.2831e-03,  8.6517e-03,  2.0562e-02,\n         5.7245e-03,  1.3288e-04, -3.1388e-02, -7.4862e-03,  1.2812e-05,\n         5.4817e-03,  1.6561e-02, -3.5099e-05,  1.4302e-02,  5.5100e-03,\n         8.0109e-03, -2.0828e-02,  2.6218e-03,  5.5258e-03,  8.2771e-03,\n         4.3478e-03,  1.7280e-02,  3.8516e-05,  5.3053e-04,  2.4896e-02,\n        -2.4664e-02, -7.6390e-03,  2.5688e-03, -2.9710e-03, -4.1264e-03,\n        -1.6960e-03,  2.2620e-03,  2.5374e-02,  1.3910e-02, -8.4825e-04,\n         1.4985e-03,  1.0969e-04,  4.4948e-03,  1.8173e-02, -2.7873e-02,\n        -1.3119e-02,  2.2612e-02,  1.1584e-02, -1.7831e-03,  1.9717e-02,\n        -2.0277e-04,  1.6129e-02,  1.1356e-02, -8.4536e-03,  1.0369e-02,\n        -2.4749e-02, -6.5893e-03,  4.1463e-04, -8.3810e-04,  1.1518e-02,\n        -2.2795e-02, -3.3748e-02,  9.6555e-03,  1.1948e-02, -1.4417e-02,\n         7.2138e-03, -4.5353e-03, -4.5611e-04,  8.8891e-03,  4.3160e-03,\n         3.8857e-03, -3.4371e-02, -7.1428e-03, -8.5142e-03, -4.9979e-03,\n         3.3383e-02, -2.1196e-02, -1.6020e-03,  1.4625e-02,  8.6971e-03,\n        -1.3028e-02, -6.6353e-03,  1.2731e-02,  5.4896e-03, -1.0384e-02,\n         8.6397e-03, -4.2818e-04, -8.7563e-03,  9.3405e-03,  5.3446e-03,\n         9.2650e-03, -1.1889e-02,  3.1683e-03, -1.6852e-02, -1.5115e-02,\n        -1.8536e-02, -7.4602e-03, -1.6525e-03, -2.8242e-02, -1.4293e-02],\n       device='cuda:0')",
-        "exp_avg_sq": "tensor([0.0019, 0.0019, 0.0018, 0.0019, 0.0023, 0.0023, 0.0012, 0.0018, 0.0022,\n        0.0023, 0.0031, 0.0029, 0.0029, 0.0029, 0.0016, 0.0019, 0.0020, 0.0029,\n        0.0016, 0.0025, 0.0023, 0.0020, 0.0020, 0.0027, 0.0020, 0.0029, 0.0042,\n        0.0022, 0.0027, 0.0025, 0.0019, 0.0022, 0.0032, 0.0025, 0.0022, 0.0019,\n        0.0018, 0.0021, 0.0029, 0.0020, 0.0022, 0.0014, 0.0024, 0.0021, 0.0020,\n        0.0026, 0.0025, 0.0022, 0.0022, 0.0019, 0.0021, 0.0019, 0.0020, 0.0024,\n        0.0024, 0.0026, 0.0040, 0.0021, 0.0027, 0.0020, 0.0021, 0.0025, 0.0023,\n        0.0031, 0.0029, 0.0021, 0.0033, 0.0028, 0.0025, 0.0024, 0.0031, 0.0018,\n        0.0031, 0.0029, 0.0020, 0.0022, 0.0020, 0.0017, 0.0019, 0.0023, 0.0020,\n        0.0020, 0.0018, 0.0025, 0.0018, 0.0019, 0.0018, 0.0026, 0.0023, 0.0021,\n        0.0023, 0.0034, 0.0023, 0.0017, 0.0026, 0.0021, 0.0019, 0.0020, 0.0018,\n        0.0024, 0.0024, 0.0017, 0.0021, 0.0018, 0.0021, 0.0028, 0.0019, 0.0021,\n        0.0034, 0.0023, 0.0025, 0.0023, 0.0019, 0.0023, 0.0021, 0.0020, 0.0024,\n        0.0023, 0.0020, 0.0036, 0.0027, 0.0030, 0.0019, 0.0020, 0.0025, 0.0025,\n        0.0040, 0.0021, 0.0021, 0.0020, 0.0023, 0.0035, 0.0026, 0.0022, 0.0022,\n        0.0021, 0.0020, 0.0021, 0.0027, 0.0032, 0.0022, 0.0024, 0.0027, 0.0025,\n        0.0025, 0.0021, 0.0017, 0.0023, 0.0023, 0.0017, 0.0022, 0.0020, 0.0033,\n        0.0034, 0.0025, 0.0026, 0.0029, 0.0019, 0.0023, 0.0025, 0.0025, 0.0034,\n        0.0027, 0.0027, 0.0024, 0.0023, 0.0017, 0.0027, 0.0028, 0.0029, 0.0015,\n        0.0017, 0.0019, 0.0020, 0.0028, 0.0029, 0.0025, 0.0019, 0.0017, 0.0021,\n        0.0031, 0.0024, 0.0023, 0.0019, 0.0018, 0.0021, 0.0025, 0.0021, 0.0029,\n        0.0032, 0.0022, 0.0028, 0.0021, 0.0025, 0.0019, 0.0020, 0.0019, 0.0027,\n        0.0022, 0.0025, 0.0022, 0.0025, 0.0033, 0.0037, 0.0024, 0.0021, 0.0027,\n        0.0022, 0.0023, 0.0024, 0.0023, 0.0016, 0.0022, 0.0034, 0.0022, 0.0035,\n        0.0021, 0.0023, 0.0021, 0.0028, 0.0023, 0.0025, 0.0017, 0.0025, 0.0020,\n        0.0020, 0.0030, 0.0021, 0.0027, 0.0023, 0.0024, 0.0023, 0.0026, 0.0020,\n        0.0024, 0.0031, 0.0021, 0.0028, 0.0021, 0.0027, 0.0022, 0.0022, 0.0021,\n        0.0029, 0.0028, 0.0023, 0.0030, 0.0027, 0.0023, 0.0025, 0.0021, 0.0019,\n        0.0036, 0.0028, 0.0021, 0.0022, 0.0025, 0.0020, 0.0018, 0.0022, 0.0019,\n        0.0023, 0.0017, 0.0032, 0.0029, 0.0023, 0.0019, 0.0016, 0.0021, 0.0018,\n        0.0024, 0.0042, 0.0022, 0.0021, 0.0019, 0.0022, 0.0020, 0.0019, 0.0023,\n        0.0020, 0.0023, 0.0029, 0.0033, 0.0027, 0.0021, 0.0024, 0.0023, 0.0020,\n        0.0031, 0.0024, 0.0020, 0.0018, 0.0019, 0.0017, 0.0022, 0.0027, 0.0023,\n        0.0019, 0.0034, 0.0025, 0.0022, 0.0022, 0.0028, 0.0018, 0.0021, 0.0026,\n        0.0021, 0.0023, 0.0020, 0.0023, 0.0021, 0.0022, 0.0015, 0.0030, 0.0035,\n        0.0025, 0.0025, 0.0024, 0.0021, 0.0021, 0.0023, 0.0031, 0.0023, 0.0022,\n        0.0026, 0.0025, 0.0027, 0.0025, 0.0029, 0.0020, 0.0039, 0.0020, 0.0030,\n        0.0022, 0.0036, 0.0030, 0.0021, 0.0020, 0.0034, 0.0024, 0.0025, 0.0023,\n        0.0019, 0.0024, 0.0029, 0.0027, 0.0020, 0.0021, 0.0024, 0.0026, 0.0018,\n        0.0023, 0.0040, 0.0026, 0.0028, 0.0024, 0.0026, 0.0023, 0.0020, 0.0024,\n        0.0017, 0.0019, 0.0019, 0.0024, 0.0031, 0.0028, 0.0028, 0.0032, 0.0021,\n        0.0029, 0.0028, 0.0032, 0.0021, 0.0028, 0.0029, 0.0030, 0.0029, 0.0028,\n        0.0020, 0.0023, 0.0033, 0.0023, 0.0022, 0.0021, 0.0023, 0.0021, 0.0029,\n        0.0025, 0.0022, 0.0023, 0.0021, 0.0028, 0.0024, 0.0028, 0.0022, 0.0022,\n        0.0021, 0.0025, 0.0024, 0.0041, 0.0020, 0.0025, 0.0024, 0.0024, 0.0023,\n        0.0028, 0.0037, 0.0020, 0.0023, 0.0022, 0.0039, 0.0027, 0.0032, 0.0021,\n        0.0023, 0.0018, 0.0020, 0.0018, 0.0020, 0.0019, 0.0032, 0.0022, 0.0026,\n        0.0029, 0.0019, 0.0022, 0.0027, 0.0027, 0.0021, 0.0035, 0.0022, 0.0024,\n        0.0014, 0.0024, 0.0027, 0.0022, 0.0027, 0.0024, 0.0028, 0.0032, 0.0019,\n        0.0019, 0.0018, 0.0031, 0.0033, 0.0025, 0.0027, 0.0026, 0.0020, 0.0030,\n        0.0038, 0.0019, 0.0021, 0.0027, 0.0026, 0.0020, 0.0025, 0.0029, 0.0028,\n        0.0020, 0.0022, 0.0020, 0.0022, 0.0021, 0.0026, 0.0020, 0.0025, 0.0027,\n        0.0018, 0.0024, 0.0024, 0.0028, 0.0016, 0.0035, 0.0020, 0.0029, 0.0027,\n        0.0017, 0.0021, 0.0022, 0.0036, 0.0017, 0.0026, 0.0026, 0.0024, 0.0024,\n        0.0023, 0.0022, 0.0021, 0.0021, 0.0019, 0.0035, 0.0017, 0.0021, 0.0022,\n        0.0020, 0.0024, 0.0019, 0.0033, 0.0020, 0.0020, 0.0021, 0.0019, 0.0024,\n        0.0022, 0.0025, 0.0026, 0.0025, 0.0023, 0.0028, 0.0023, 0.0027, 0.0016,\n        0.0026, 0.0015, 0.0028, 0.0019, 0.0023, 0.0020, 0.0024, 0.0029, 0.0020,\n        0.0021, 0.0019, 0.0026, 0.0024, 0.0018, 0.0021, 0.0024, 0.0025, 0.0024,\n        0.0018, 0.0016, 0.0020, 0.0028, 0.0022, 0.0025, 0.0023, 0.0018, 0.0026,\n        0.0020, 0.0023, 0.0022, 0.0022, 0.0034, 0.0025, 0.0020, 0.0025, 0.0017,\n        0.0015, 0.0026, 0.0024, 0.0023, 0.0023, 0.0020, 0.0023, 0.0024, 0.0025,\n        0.0024, 0.0022, 0.0026, 0.0024, 0.0018, 0.0027, 0.0019, 0.0021, 0.0020,\n        0.0017, 0.0035, 0.0027, 0.0022, 0.0027, 0.0024, 0.0021, 0.0029, 0.0024,\n        0.0022, 0.0038, 0.0018, 0.0031, 0.0025, 0.0032, 0.0029, 0.0024, 0.0020,\n        0.0024, 0.0029, 0.0019, 0.0020, 0.0029, 0.0020, 0.0036, 0.0029, 0.0024,\n        0.0020, 0.0023, 0.0019, 0.0024, 0.0024, 0.0021, 0.0018, 0.0023, 0.0031,\n        0.0024, 0.0019, 0.0021, 0.0023, 0.0021, 0.0025, 0.0023, 0.0023, 0.0026,\n        0.0018, 0.0029, 0.0031, 0.0019, 0.0050, 0.0022, 0.0032, 0.0021, 0.0031,\n        0.0022, 0.0020, 0.0024, 0.0019, 0.0024, 0.0027, 0.0013, 0.0024, 0.0025,\n        0.0024, 0.0034, 0.0020, 0.0028, 0.0028, 0.0022, 0.0027, 0.0027, 0.0021,\n        0.0022], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([ 7.5002e-03,  6.1666e-03, -4.9600e-03,  6.1611e-03,  4.2384e-02,\n        -1.9220e-02, -8.3844e-03,  4.7075e-04,  1.0635e-02, -1.0766e-02,\n        -8.4952e-04,  1.1064e-03, -7.1118e-03, -1.7303e-02, -1.9323e-02,\n        -1.2425e-02,  1.0077e-02, -4.0970e-02,  1.6759e-03, -2.3874e-02,\n         1.6497e-02, -1.2889e-04,  6.9716e-03,  2.6505e-02,  1.5302e-03,\n         1.1848e-02,  1.9951e-02, -8.8258e-03,  3.5513e-03,  1.4729e-02,\n         1.0189e-02, -4.1829e-03,  4.6535e-03, -2.6548e-02,  1.1126e-03,\n        -2.9267e-02, -6.5974e-03,  1.0647e-02, -3.9801e-03, -8.8817e-05,\n        -1.8313e-02,  6.3268e-03,  5.1648e-03,  3.6987e-03, -8.3055e-03,\n        -3.9887e-03, -3.4601e-03,  1.9262e-02,  1.4138e-02, -1.7778e-02,\n        -3.2670e-03,  3.1757e-02,  3.8606e-03, -3.3370e-03,  3.0356e-02,\n        -1.3397e-02, -2.5703e-03,  1.4614e-02, -3.3291e-02,  8.3999e-04,\n         1.1530e-02, -1.9905e-02,  8.5553e-03,  1.5821e-02,  1.2836e-02,\n        -5.8094e-03, -2.4451e-02,  1.0649e-02,  3.0301e-03,  2.4705e-03,\n        -4.2230e-02, -1.3881e-02, -1.5045e-02, -5.2605e-03,  1.8229e-02,\n         5.9738e-03,  8.7830e-03,  6.5329e-03, -6.0538e-03,  1.3049e-02,\n        -6.7696e-03, -6.8786e-04,  2.6906e-02,  2.2707e-02, -1.9941e-02,\n        -1.1775e-02, -1.5720e-02, -1.2808e-02, -3.2262e-03,  2.2056e-02,\n         9.3135e-03, -2.3001e-03,  4.6321e-03,  1.6632e-02, -1.7077e-02,\n         2.4845e-03,  7.0338e-03, -2.0788e-03,  4.1306e-03, -1.8734e-02,\n         4.3794e-03,  1.5470e-02, -2.0499e-03,  7.6800e-03,  2.3201e-03,\n         8.2508e-03,  2.5984e-02,  1.7805e-02, -9.6886e-04, -9.4811e-03,\n        -2.1282e-02,  1.0057e-02,  1.6776e-02,  2.8346e-03, -4.4798e-02,\n         1.0172e-02,  6.5314e-03,  1.2147e-02, -7.4831e-03, -8.4042e-03,\n         1.0060e-02,  5.8784e-03, -5.0699e-03, -2.3155e-02,  1.2979e-02,\n        -4.0700e-02,  3.8160e-02, -1.5794e-03, -1.9136e-02,  8.3348e-03,\n        -5.4544e-03, -6.8930e-03,  1.8212e-03,  3.0208e-03, -7.2003e-03,\n        -3.7202e-03,  3.5775e-03,  1.2043e-02, -1.8218e-02,  1.0326e-02,\n        -4.7054e-03,  2.6481e-02, -2.2354e-03, -9.7157e-03, -2.0954e-02,\n        -2.0589e-02, -3.7296e-03,  8.4174e-03,  7.1278e-03,  1.8315e-03,\n        -6.1332e-03,  2.1701e-02,  1.7961e-03, -3.5695e-02, -3.6253e-03,\n         6.3861e-03, -8.7802e-03,  2.5661e-02, -1.4770e-02,  2.3363e-03,\n         4.8627e-06, -3.4831e-03, -1.7585e-02, -1.3600e-02, -4.7705e-03,\n         3.2852e-03, -4.3559e-03,  1.2411e-02, -6.3844e-03,  1.4594e-02,\n        -2.1835e-02,  2.2034e-02,  7.4159e-03, -1.9344e-02, -4.2616e-03,\n         5.3107e-03,  1.8573e-02,  3.3974e-03,  8.9747e-03, -2.9110e-02,\n        -6.1603e-04,  2.6851e-02,  1.0675e-03,  3.8335e-03, -2.1870e-02,\n        -7.2596e-03,  1.3847e-02,  3.5364e-04, -2.6820e-02,  4.1513e-03,\n        -5.5434e-03,  8.5315e-03,  9.0884e-03,  6.4947e-03, -2.7338e-03,\n        -2.1389e-02,  1.0514e-02, -3.7026e-03, -7.4370e-03, -1.8809e-02,\n        -1.1799e-02, -2.1281e-02, -1.3515e-02,  2.7394e-03, -1.8285e-02,\n         2.8042e-03,  6.4321e-04, -3.2111e-02,  1.2037e-02,  2.1382e-03,\n        -3.2882e-03, -8.3480e-03, -4.7763e-03,  1.0980e-02, -6.0826e-03,\n         1.2728e-02,  2.0775e-03, -4.6533e-03, -7.1669e-04,  6.6817e-03,\n         8.3459e-03, -5.2075e-03,  1.3601e-02, -7.4524e-03,  6.3202e-03,\n        -3.6381e-03, -1.1919e-03,  2.1841e-02, -2.0557e-02,  2.1676e-02,\n         1.1317e-02, -2.5531e-02,  5.8963e-03,  1.7122e-02, -7.2185e-03,\n         5.5723e-04,  4.0023e-03,  7.6886e-03, -6.5542e-03, -1.0972e-04,\n         1.2319e-02,  1.0818e-02,  8.8110e-04,  5.4517e-03, -1.4672e-02,\n         7.6637e-03, -2.2257e-02, -2.4268e-03,  6.0236e-03,  5.7640e-03,\n        -9.3002e-03,  6.5061e-03,  9.9962e-03,  7.4713e-03, -2.2215e-02,\n         5.7826e-03,  8.8195e-03, -1.1466e-02,  8.2011e-03,  7.4601e-04,\n         2.2986e-03, -2.8695e-03, -1.6565e-03,  2.7637e-02,  1.7682e-02,\n        -1.1140e-02,  8.0441e-03, -1.1175e-02,  1.1836e-02,  3.4923e-03,\n         1.2227e-02, -1.3963e-02,  9.4421e-03,  8.7784e-03, -4.5916e-03,\n         1.3755e-02, -9.8096e-03, -1.0324e-02, -1.0254e-03,  1.3824e-02,\n        -5.8864e-03,  2.8100e-02,  1.9033e-02, -1.7944e-03,  5.0956e-04,\n         1.2556e-02,  3.5404e-02, -1.1972e-02, -4.4883e-03, -6.9917e-04,\n         2.7119e-03, -4.3158e-02,  1.9065e-02,  5.5506e-03,  1.8930e-03,\n        -1.5561e-02,  2.4375e-02, -3.1385e-03,  2.1630e-02,  2.1765e-03,\n        -1.1839e-02,  4.0120e-03,  3.2524e-02, -1.5400e-02,  3.9087e-03,\n         9.0975e-03, -7.9342e-03,  8.9844e-03,  1.1798e-02, -6.3013e-03,\n        -6.8291e-03, -3.5375e-03,  6.0614e-03,  3.3835e-02, -1.0973e-02,\n         6.9588e-03,  4.2893e-03, -1.0467e-03,  1.8468e-02, -3.2033e-03,\n        -1.6772e-02, -4.8663e-04,  2.1358e-02,  8.8287e-03, -1.2741e-02,\n        -9.2074e-03,  3.4563e-03,  1.7435e-02,  1.8690e-03,  2.9876e-04,\n         1.0413e-04, -1.0112e-02,  1.6502e-02,  2.1767e-03, -6.0951e-03,\n         1.9795e-02,  5.1981e-03,  6.8298e-03, -9.1880e-04, -6.1209e-03,\n         1.9639e-02, -7.0725e-03, -8.4911e-04, -3.1936e-03, -8.1469e-03,\n         2.2903e-03, -7.2077e-03, -1.2629e-03, -1.6029e-03, -3.0349e-02,\n         6.3177e-03, -2.2232e-02, -7.8631e-04, -1.0223e-02, -1.4878e-02,\n         1.2067e-03,  1.0516e-02,  7.4746e-03,  2.3452e-04,  1.0757e-02,\n        -6.7234e-03, -1.9288e-02, -6.6883e-04, -1.4599e-02,  2.7748e-03,\n        -2.7878e-02,  2.2437e-02, -2.1864e-02, -1.3880e-02, -9.6712e-03,\n        -1.9037e-03,  5.7162e-03,  1.7645e-02, -9.6087e-03, -4.7373e-03,\n        -1.1607e-02,  6.1116e-03, -6.8908e-03, -5.1185e-03,  1.2684e-02,\n         5.2543e-04,  5.7019e-03, -1.6155e-02,  8.7226e-04, -8.4820e-03,\n        -4.5337e-03,  1.0811e-02,  1.4901e-02, -1.3097e-02, -1.8544e-02,\n         1.3208e-02,  3.0735e-03,  2.6452e-02, -3.4206e-02,  1.4948e-02,\n        -2.2616e-03, -1.4811e-03, -1.4967e-02,  3.1700e-03, -5.1466e-02,\n         9.4068e-03,  3.6288e-03,  5.1295e-03,  6.7509e-03,  5.6759e-03,\n         1.4958e-02,  7.1466e-03, -8.6712e-04,  1.8486e-02,  7.7097e-03,\n         4.0422e-04,  1.0883e-02,  8.9003e-03,  8.4420e-03, -1.3882e-02,\n         1.2395e-02,  4.8072e-03, -4.1003e-03, -2.3001e-03, -8.5254e-03,\n         3.1507e-02,  4.4109e-03, -4.6974e-03,  1.4503e-02, -1.9600e-03,\n        -1.5566e-02, -1.4285e-03,  3.3898e-03,  2.0972e-02, -3.7452e-02,\n         9.9496e-03,  1.6657e-02,  1.4277e-02, -4.9693e-04,  1.3667e-02,\n         9.2383e-03,  1.0335e-02, -4.0814e-03,  2.1612e-02, -1.1772e-02,\n         5.7829e-03,  1.9121e-03,  5.4433e-04, -2.1631e-02, -1.0545e-02,\n        -1.8550e-02, -1.0206e-02,  6.1815e-03,  2.6431e-04,  6.5531e-03,\n        -4.3925e-02, -1.4396e-02, -7.9888e-03, -5.1216e-03,  1.7811e-02,\n         9.3742e-03, -1.0087e-02, -9.3537e-03,  1.0426e-03,  5.4855e-03,\n        -8.4249e-03, -7.2711e-04,  1.8274e-02,  8.7232e-03,  3.8414e-03,\n         1.2546e-02,  1.0764e-02,  1.6020e-02, -2.5126e-03,  5.8987e-03,\n         7.1405e-03, -4.8120e-03, -3.5610e-03, -5.2019e-03, -5.7160e-04,\n        -5.4134e-03,  2.1359e-02,  7.2265e-03, -9.2278e-03, -2.0810e-03,\n        -1.0700e-02,  1.0448e-03,  4.2769e-03,  4.6024e-03, -2.2860e-02,\n         2.1438e-02,  1.9457e-02,  3.5407e-03, -1.2072e-02, -3.5997e-02,\n         4.1577e-03, -5.7400e-03, -1.5299e-02,  1.0582e-02,  4.2529e-03,\n         4.8344e-03,  9.6877e-03, -9.8788e-03,  6.9282e-03,  1.5047e-02,\n        -1.8485e-02,  2.0370e-03, -9.5045e-03, -1.1592e-02,  2.2464e-02,\n        -1.9486e-02,  1.1776e-02,  3.4814e-03,  1.8949e-02,  1.1111e-02,\n         7.3057e-03, -5.6502e-03,  8.0085e-03,  1.0480e-02, -1.3336e-02,\n        -1.5076e-02, -3.6794e-03, -1.1929e-02,  1.0319e-02,  1.8013e-02,\n         1.2966e-02,  4.3545e-03,  1.3252e-02,  1.5814e-02,  9.4098e-03,\n        -6.4499e-03,  4.0679e-03, -4.1531e-03,  1.1122e-02,  1.0362e-03,\n        -8.7823e-03,  7.8166e-03, -2.3918e-03,  9.2952e-04,  2.0623e-03,\n         5.4402e-03,  2.4221e-02,  1.1807e-02, -3.8971e-03, -3.0831e-02,\n        -8.1170e-04, -1.7695e-02,  1.5276e-02, -9.0631e-03, -6.0323e-03,\n        -5.6246e-03, -4.6081e-02,  1.5295e-02, -3.0662e-03,  1.4421e-03,\n         3.1950e-03, -1.6876e-02,  1.4467e-03,  1.9590e-02, -3.2373e-02,\n         8.8691e-03,  1.6429e-02, -3.3556e-03,  1.1395e-02,  5.2048e-03,\n        -1.0511e-02, -1.3843e-02, -3.9300e-02,  9.6730e-03,  1.2379e-02,\n         2.5041e-04,  1.8809e-02, -2.1314e-03, -2.1516e-02,  2.0708e-03,\n        -3.0634e-03, -8.5988e-03, -1.2090e-02, -1.2553e-02, -1.0845e-02,\n         1.0688e-03,  5.1341e-03,  1.4643e-02, -7.8537e-04,  4.6313e-03,\n        -1.3006e-02, -4.9244e-03, -1.0704e-02, -4.5927e-03,  1.6731e-03,\n        -4.8729e-03, -1.1680e-02, -2.1329e-03, -1.0017e-03,  6.3447e-03,\n        -1.9672e-03,  2.6279e-03, -1.8901e-03,  4.0267e-03,  3.6861e-03,\n         1.4563e-02, -3.3208e-03,  4.6089e-03, -8.4111e-03, -1.5482e-02,\n         1.3915e-02, -1.1742e-02,  1.4256e-02,  2.0253e-02, -4.6216e-03,\n        -2.3277e-02, -1.2902e-03,  7.1385e-03,  3.1895e-03,  2.5037e-03,\n         3.0609e-03, -2.2982e-03, -1.6950e-02,  1.0686e-02,  1.2337e-02,\n         5.1010e-03,  3.3296e-03, -8.7820e-03,  4.1231e-04,  1.4284e-02,\n         5.2430e-03,  1.3779e-02,  1.1006e-02,  3.4085e-03,  9.4340e-04,\n         1.3037e-02, -6.5691e-03, -5.1768e-03,  5.0331e-03, -3.0340e-02,\n         1.9957e-02, -2.3842e-03,  1.0488e-02,  8.7999e-04,  8.6451e-03,\n         9.6842e-03,  9.2201e-04,  7.2388e-03, -4.3417e-03,  1.7046e-02],\n       device='cuda:0')",
+        "exp_avg_sq": "tensor([0.0017, 0.0019, 0.0016, 0.0015, 0.0021, 0.0019, 0.0011, 0.0017, 0.0017,\n        0.0018, 0.0026, 0.0026, 0.0028, 0.0024, 0.0014, 0.0016, 0.0017, 0.0026,\n        0.0012, 0.0022, 0.0022, 0.0015, 0.0016, 0.0024, 0.0017, 0.0023, 0.0037,\n        0.0018, 0.0021, 0.0022, 0.0014, 0.0016, 0.0030, 0.0022, 0.0019, 0.0015,\n        0.0013, 0.0020, 0.0025, 0.0017, 0.0016, 0.0011, 0.0021, 0.0017, 0.0017,\n        0.0023, 0.0021, 0.0019, 0.0018, 0.0016, 0.0016, 0.0015, 0.0017, 0.0022,\n        0.0020, 0.0021, 0.0037, 0.0017, 0.0025, 0.0015, 0.0019, 0.0020, 0.0017,\n        0.0025, 0.0025, 0.0017, 0.0032, 0.0023, 0.0021, 0.0019, 0.0028, 0.0015,\n        0.0025, 0.0022, 0.0020, 0.0017, 0.0017, 0.0014, 0.0016, 0.0019, 0.0016,\n        0.0015, 0.0016, 0.0020, 0.0015, 0.0015, 0.0016, 0.0025, 0.0020, 0.0017,\n        0.0018, 0.0027, 0.0017, 0.0016, 0.0027, 0.0018, 0.0013, 0.0016, 0.0015,\n        0.0022, 0.0020, 0.0013, 0.0016, 0.0015, 0.0017, 0.0024, 0.0015, 0.0018,\n        0.0032, 0.0019, 0.0021, 0.0018, 0.0016, 0.0019, 0.0020, 0.0017, 0.0018,\n        0.0018, 0.0017, 0.0031, 0.0027, 0.0022, 0.0017, 0.0017, 0.0025, 0.0024,\n        0.0036, 0.0018, 0.0017, 0.0016, 0.0021, 0.0031, 0.0023, 0.0017, 0.0019,\n        0.0017, 0.0019, 0.0019, 0.0024, 0.0025, 0.0018, 0.0019, 0.0023, 0.0021,\n        0.0021, 0.0016, 0.0014, 0.0018, 0.0019, 0.0015, 0.0016, 0.0018, 0.0027,\n        0.0028, 0.0020, 0.0021, 0.0025, 0.0017, 0.0020, 0.0020, 0.0021, 0.0030,\n        0.0022, 0.0022, 0.0022, 0.0017, 0.0013, 0.0026, 0.0024, 0.0028, 0.0012,\n        0.0015, 0.0017, 0.0018, 0.0025, 0.0024, 0.0021, 0.0014, 0.0015, 0.0017,\n        0.0025, 0.0022, 0.0020, 0.0015, 0.0015, 0.0016, 0.0021, 0.0018, 0.0025,\n        0.0026, 0.0020, 0.0025, 0.0017, 0.0022, 0.0016, 0.0018, 0.0015, 0.0025,\n        0.0016, 0.0021, 0.0019, 0.0020, 0.0032, 0.0033, 0.0020, 0.0017, 0.0020,\n        0.0021, 0.0021, 0.0021, 0.0019, 0.0013, 0.0018, 0.0031, 0.0016, 0.0027,\n        0.0018, 0.0018, 0.0018, 0.0021, 0.0020, 0.0020, 0.0012, 0.0021, 0.0016,\n        0.0018, 0.0025, 0.0015, 0.0023, 0.0020, 0.0020, 0.0021, 0.0019, 0.0016,\n        0.0022, 0.0026, 0.0017, 0.0023, 0.0017, 0.0023, 0.0017, 0.0018, 0.0018,\n        0.0025, 0.0023, 0.0019, 0.0024, 0.0023, 0.0018, 0.0020, 0.0017, 0.0014,\n        0.0035, 0.0021, 0.0015, 0.0017, 0.0022, 0.0017, 0.0015, 0.0017, 0.0015,\n        0.0019, 0.0014, 0.0029, 0.0025, 0.0018, 0.0016, 0.0013, 0.0017, 0.0016,\n        0.0019, 0.0040, 0.0018, 0.0018, 0.0015, 0.0020, 0.0017, 0.0016, 0.0018,\n        0.0017, 0.0016, 0.0025, 0.0027, 0.0024, 0.0017, 0.0019, 0.0019, 0.0015,\n        0.0027, 0.0017, 0.0017, 0.0017, 0.0017, 0.0014, 0.0018, 0.0022, 0.0019,\n        0.0015, 0.0030, 0.0020, 0.0017, 0.0017, 0.0025, 0.0014, 0.0018, 0.0022,\n        0.0017, 0.0018, 0.0017, 0.0021, 0.0015, 0.0019, 0.0012, 0.0026, 0.0028,\n        0.0020, 0.0023, 0.0020, 0.0019, 0.0018, 0.0021, 0.0024, 0.0020, 0.0019,\n        0.0021, 0.0024, 0.0023, 0.0021, 0.0028, 0.0016, 0.0038, 0.0015, 0.0025,\n        0.0018, 0.0028, 0.0027, 0.0019, 0.0015, 0.0028, 0.0020, 0.0019, 0.0020,\n        0.0014, 0.0020, 0.0026, 0.0023, 0.0015, 0.0016, 0.0019, 0.0022, 0.0015,\n        0.0021, 0.0035, 0.0022, 0.0025, 0.0017, 0.0021, 0.0018, 0.0018, 0.0021,\n        0.0014, 0.0015, 0.0017, 0.0021, 0.0028, 0.0023, 0.0024, 0.0028, 0.0019,\n        0.0026, 0.0022, 0.0025, 0.0017, 0.0030, 0.0022, 0.0024, 0.0028, 0.0023,\n        0.0017, 0.0019, 0.0024, 0.0018, 0.0018, 0.0017, 0.0020, 0.0016, 0.0026,\n        0.0025, 0.0018, 0.0019, 0.0017, 0.0020, 0.0020, 0.0025, 0.0017, 0.0018,\n        0.0017, 0.0024, 0.0020, 0.0042, 0.0017, 0.0020, 0.0020, 0.0018, 0.0020,\n        0.0026, 0.0032, 0.0015, 0.0019, 0.0018, 0.0034, 0.0021, 0.0031, 0.0018,\n        0.0020, 0.0015, 0.0016, 0.0015, 0.0016, 0.0017, 0.0028, 0.0018, 0.0023,\n        0.0029, 0.0017, 0.0018, 0.0025, 0.0024, 0.0018, 0.0030, 0.0019, 0.0019,\n        0.0011, 0.0021, 0.0022, 0.0019, 0.0022, 0.0019, 0.0026, 0.0032, 0.0015,\n        0.0015, 0.0014, 0.0031, 0.0031, 0.0024, 0.0023, 0.0020, 0.0016, 0.0026,\n        0.0038, 0.0015, 0.0016, 0.0023, 0.0018, 0.0018, 0.0020, 0.0025, 0.0023,\n        0.0017, 0.0019, 0.0017, 0.0018, 0.0019, 0.0022, 0.0018, 0.0020, 0.0023,\n        0.0016, 0.0022, 0.0021, 0.0024, 0.0013, 0.0030, 0.0018, 0.0022, 0.0024,\n        0.0014, 0.0017, 0.0019, 0.0030, 0.0014, 0.0022, 0.0020, 0.0020, 0.0022,\n        0.0021, 0.0019, 0.0017, 0.0017, 0.0016, 0.0030, 0.0015, 0.0017, 0.0018,\n        0.0015, 0.0019, 0.0014, 0.0029, 0.0019, 0.0015, 0.0018, 0.0015, 0.0022,\n        0.0020, 0.0022, 0.0022, 0.0020, 0.0017, 0.0023, 0.0019, 0.0023, 0.0013,\n        0.0021, 0.0011, 0.0026, 0.0015, 0.0018, 0.0019, 0.0020, 0.0025, 0.0019,\n        0.0017, 0.0016, 0.0020, 0.0019, 0.0015, 0.0017, 0.0021, 0.0021, 0.0019,\n        0.0016, 0.0012, 0.0017, 0.0023, 0.0017, 0.0021, 0.0018, 0.0013, 0.0024,\n        0.0015, 0.0020, 0.0017, 0.0019, 0.0024, 0.0021, 0.0018, 0.0021, 0.0014,\n        0.0013, 0.0019, 0.0019, 0.0021, 0.0018, 0.0018, 0.0020, 0.0023, 0.0020,\n        0.0021, 0.0018, 0.0020, 0.0022, 0.0014, 0.0025, 0.0017, 0.0016, 0.0018,\n        0.0013, 0.0029, 0.0025, 0.0019, 0.0022, 0.0023, 0.0020, 0.0024, 0.0020,\n        0.0021, 0.0036, 0.0016, 0.0028, 0.0022, 0.0027, 0.0025, 0.0019, 0.0018,\n        0.0020, 0.0021, 0.0014, 0.0016, 0.0025, 0.0019, 0.0032, 0.0024, 0.0020,\n        0.0015, 0.0018, 0.0015, 0.0020, 0.0019, 0.0017, 0.0013, 0.0019, 0.0029,\n        0.0021, 0.0018, 0.0018, 0.0021, 0.0017, 0.0022, 0.0019, 0.0019, 0.0021,\n        0.0016, 0.0024, 0.0025, 0.0014, 0.0044, 0.0018, 0.0028, 0.0017, 0.0029,\n        0.0021, 0.0017, 0.0022, 0.0015, 0.0019, 0.0024, 0.0011, 0.0017, 0.0021,\n        0.0022, 0.0031, 0.0017, 0.0023, 0.0024, 0.0018, 0.0025, 0.0025, 0.0015,\n        0.0017], device='cuda:0')"
       },
       "2": {
-        "step": "tensor(2504.)",
-        "exp_avg": "tensor([[ 3.3157e-05,  9.0172e-05,  8.2628e-05,  ...,  5.1896e-05,\n          9.1822e-05, -2.2610e-04],\n        [-4.5172e-04, -1.2079e-04,  1.8322e-04,  ..., -3.3412e-05,\n          1.8097e-04,  1.7297e-04],\n        [ 4.6973e-04, -8.3150e-05, -9.6866e-04,  ..., -3.8514e-05,\n          1.4621e-04, -6.8942e-05],\n        ...,\n        [-5.8888e-04, -3.6048e-04, -3.6673e-05,  ..., -1.5665e-05,\n         -4.0896e-04,  5.4799e-04],\n        [-1.2193e-03, -3.9815e-04,  3.8778e-04,  ..., -5.7737e-05,\n         -7.0575e-05,  2.8701e-05],\n        [ 1.0101e-04,  1.1042e-04,  2.3041e-04,  ...,  1.5737e-05,\n         -3.5228e-04, -4.9031e-05]], device='cuda:0')",
-        "exp_avg_sq": "tensor([[8.4538e-07, 6.2770e-07, 8.0637e-07,  ..., 3.5103e-07, 8.9550e-07,\n         1.4680e-06],\n        [1.2847e-06, 1.6966e-06, 1.4330e-06,  ..., 9.8359e-07, 2.4769e-06,\n         2.1182e-06],\n        [1.5711e-06, 1.8740e-06, 1.7353e-06,  ..., 1.0199e-06, 1.4728e-06,\n         1.2872e-06],\n        ...,\n        [1.6384e-06, 1.4097e-06, 1.7026e-06,  ..., 1.1115e-06, 1.9571e-06,\n         3.8118e-06],\n        [1.8098e-06, 2.3717e-06, 1.8668e-06,  ..., 8.7579e-07, 1.8769e-06,\n         1.9211e-06],\n        [2.0767e-06, 8.3075e-07, 2.4246e-06,  ..., 1.2058e-06, 1.8157e-06,\n         2.2155e-06]], device='cuda:0')"
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[-2.0531e-04,  2.6832e-04, -6.0653e-05,  ..., -6.8946e-05,\n          1.3081e-04, -3.1265e-04],\n        [-1.5010e-04,  2.7194e-05, -4.7230e-05,  ...,  1.9473e-05,\n         -2.3948e-04, -3.1832e-04],\n        [-3.2661e-05, -2.5834e-04,  5.6320e-05,  ...,  7.7295e-05,\n         -8.1635e-04,  8.1857e-05],\n        ...,\n        [ 6.3526e-05, -2.0283e-05, -2.0169e-04,  ..., -9.2684e-05,\n          2.1288e-04, -1.7126e-04],\n        [-1.1934e-04,  1.8057e-04,  3.5367e-04,  ...,  1.4586e-05,\n          9.6794e-06, -1.6967e-04],\n        [ 1.1826e-05,  4.2725e-04, -3.2022e-04,  ...,  7.0693e-05,\n         -4.3537e-04, -3.5973e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[5.2474e-07, 4.8824e-07, 5.0117e-07,  ..., 2.0473e-07, 5.6365e-07,\n         1.4658e-06],\n        [9.0473e-07, 1.4383e-06, 8.6538e-07,  ..., 5.9647e-07, 1.7928e-06,\n         1.6739e-06],\n        [1.1156e-06, 1.5511e-06, 1.1785e-06,  ..., 6.4719e-07, 1.0678e-06,\n         9.4882e-07],\n        ...,\n        [1.1252e-06, 1.0989e-06, 1.0603e-06,  ..., 7.4304e-07, 1.2404e-06,\n         2.8223e-06],\n        [1.1438e-06, 2.1048e-06, 1.1909e-06,  ..., 4.8723e-07, 1.1319e-06,\n         1.8908e-06],\n        [1.5152e-06, 6.8419e-07, 1.6502e-06,  ..., 7.0794e-07, 1.0720e-06,\n         1.9125e-06]], device='cuda:0')"
       }
     },
     "param_groups": [
       {
-        "lr": 0.0009046039886902864,
+        "lr": 0.0007940987335200904,
         "name": "scale_256",
         "betas": [
           0.9,
@@ -43,7 +43,7 @@
         ]
       },
       {
-        "lr": 0.0009046039886902864,
+        "lr": 0.0007940987335200904,
         "name": "scale_512",
         "betas": [
           0.9,
@@ -66,7 +66,7 @@
         ]
       },
       {
-        "lr": 0.0009046039886902864,
+        "lr": 0.0007940987335200904,
         "name": "scale_768",
         "betas": [
           0.9,
@@ -89,7 +89,7 @@
         ]
       },
       {
-        "lr": 0.0009046039886902864,
+        "lr": 0.0007940987335200904,
         "name": "scale_1024",
         "betas": [
           0.9,
@@ -112,7 +112,7 @@
         ]
       },
       {
-        "lr": 0.0009046039886902864,
+        "lr": 0.0007940987335200904,
         "name": "scale_1280",
         "betas": [
           0.9,
@@ -135,7 +135,7 @@
         ]
       },
       {
-        "lr": 0.0004523497400965494,
+        "lr": 0.00039715242044697206,
         "name": "fusion",
         "betas": [
           0.9,
@@ -195,7 +195,7 @@
     "T_i": 10,
     "T_mult": 2,
     "eta_min": 1e-06,
-    "T_cur": 2,
+    "T_cur": 3,
     "base_lrs": [
       0.001,
       0.001,
@@ -204,24 +204,24 @@
       0.001,
       0.0005
     ],
-    "last_epoch": 2,
+    "last_epoch": 3,
     "_step_count": 0,
     "_is_initial": false,
     "_get_lr_called_within_step": false,
     "_last_lr": [
-      0.0009046039886902864,
-      0.0009046039886902864,
-      0.0009046039886902864,
-      0.0009046039886902864,
-      0.0009046039886902864,
-      0.0004523497400965494
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.00039715242044697206
     ]
   },
   "metrics": {
-    "best_val_acc": 71.226,
-    "best_epoch": 1,
+    "best_val_acc": 71.666,
+    "best_epoch": 2,
     "scale_accuracies": {
-      "256": 71.226
+      "256": 71.666
     }
   },
   "train_config": {