Upload weights and configs - David-fully_shared-weighted_sum - Run 20251012_135249

weights/David-fully_shared-weighted_sum/20251012_135249/MODEL_SUMMARY.txt

@@ -3,24 +3,24 @@
 ║                      DAVID MODEL SUMMARY                      ║
 ╠══════════════════════════════════════════════════════════════╣
 ║                                                               ║
-║  🎯 VALIDATION ACCURACY: 69.62%                          ║
+║  🎯 VALIDATION ACCURACY: 70.14%                          ║
 ║                                                               ║
 ╚══════════════════════════════════════════════════════════════╝
 
 MODEL: David-fully_shared-weighted_sum
 RUN ID: 20251012_135249
-BEST EPOCH: 2/10
+BEST EPOCH: 3/10
 
 ═══════════════════════════════════════════════════════════════
 
 📊 PERFORMANCE BREAKDOWN
 
-Final Training Accuracy:  69.48%
-Best Validation Accuracy: 69.62%
+Final Training Accuracy:  70.34%
+Best Validation Accuracy: 70.14%
 
 Per-Scale Accuracies:
-  • Scale  256: 68.83%
-  • Scale  512: 69.57%
+  • Scale  256: 69.54%
+  • Scale  512: 70.03%
 
 ═══════════════════════════════════════════════════════════════
 
@@ -40,17 +40,18 @@ Parameters:    656,898
 Epoch | Train Acc | Val Acc  | Learning Rate
 ------|-----------|----------|--------------
     1 |    63.40% |   68.06% | 9.76e-04
-    2 |    69.48% |   69.62% 👑 | 9.05e-04
+    2 |    69.48% |   69.62% | 9.05e-04
+    3 |    70.34% |   70.14% 👑 | 7.94e-04
 
 ═══════════════════════════════════════════════════════════════
 
 📁 FILES
 
-Best Model:    best_model_acc69.62.safetensors
+Best Model:    best_model_acc70.14.safetensors
 Config:        david_config.json
 Training Cfg:  train_config.json
 History:       training_history.json
 
 ═══════════════════════════════════════════════════════════════
 
-Generated: 2025-10-12 13:56:13
+Generated: 2025-10-12 13:57:56

weights/David-fully_shared-weighted_sum/20251012_135249/best_model_acc70.14.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f6391b526ade6b4350f2307a7ab08f82c51e2a789ce4b891f2c37149259fa4
+size 2628344

weights/David-fully_shared-weighted_sum/20251012_135249/best_model_acc70.14_metadata.json

@@ -0,0 +1,251 @@
+{
+  "epoch": 2,
+  "optimizer_state_dict": {
+    "state": {
+      "0": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[ 1.7448e-04, -4.8809e-03, -1.8495e-04,  ..., -1.9395e-04,\n         -8.3969e-05, -5.8805e-05],\n        [-1.2346e-04, -3.1065e-03, -2.0100e-03,  ..., -1.0883e-03,\n         -4.3606e-04, -2.4338e-03],\n        [ 6.5367e-05, -1.2240e-03,  1.7775e-04,  ...,  6.4026e-04,\n         -1.6947e-04, -2.4394e-04],\n        ...,\n        [-6.8234e-04, -2.7625e-03, -3.6629e-04,  ...,  4.7172e-05,\n          2.1757e-04, -5.1777e-05],\n        [-1.8593e-04, -6.2607e-04, -4.6365e-04,  ...,  5.1322e-04,\n          5.5338e-04, -4.9640e-04],\n        [-7.0405e-04, -1.9380e-03,  1.0305e-03,  ...,  3.6356e-04,\n         -6.3794e-05,  9.8156e-05]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[6.9370e-06, 3.2780e-05, 1.4851e-05,  ..., 5.6356e-06, 3.1386e-06,\n         7.1158e-06],\n        [8.0753e-06, 3.2987e-05, 2.4073e-05,  ..., 1.0438e-05, 2.9388e-06,\n         1.3223e-05],\n        [7.4407e-06, 4.6800e-05, 1.8279e-05,  ..., 4.7452e-06, 2.5969e-06,\n         3.6221e-06],\n        ...,\n        [6.6446e-06, 2.7955e-05, 1.4523e-05,  ..., 3.2907e-06, 1.8194e-06,\n         2.8980e-06],\n        [5.9549e-06, 3.4965e-05, 1.3442e-05,  ..., 5.0011e-06, 3.0061e-06,\n         5.6436e-06],\n        [5.4520e-06, 5.4596e-05, 1.3267e-05,  ..., 3.4816e-06, 2.4878e-06,\n         2.8721e-06]], device='cuda:0')"
+      },
+      "1": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([ 2.1909e-02, -4.0367e-02,  2.3634e-02, -1.1208e-02,  1.3224e-02,\n         7.4066e-03, -3.2574e-02,  9.6779e-03,  2.5852e-02,  1.3851e-02,\n        -2.7764e-02,  1.5100e-02, -2.1058e-03, -8.6947e-03,  1.3905e-02,\n         7.2412e-03,  1.8327e-02,  6.1515e-03, -5.8562e-03, -1.2230e-02,\n        -1.1005e-02, -3.1954e-02,  3.0161e-02,  4.0579e-03,  1.4330e-02,\n        -1.1709e-02, -3.6856e-03, -3.2731e-02, -4.3399e-03,  6.7614e-03,\n         8.2980e-03,  6.1146e-02, -9.2499e-03,  8.0002e-03,  6.1083e-03,\n        -2.0457e-02, -7.4349e-03,  2.6419e-02,  1.8807e-02, -1.4217e-02,\n         1.3726e-02, -1.5311e-02,  8.0120e-03, -6.2022e-03, -1.0418e-02,\n         5.4429e-02,  9.1602e-03,  1.9926e-02,  8.6272e-03,  2.8791e-03,\n        -2.1483e-02,  4.4225e-02, -1.5325e-02,  1.8660e-02,  3.5077e-02,\n         2.2682e-02,  1.4871e-03,  3.6448e-02,  6.1576e-04,  2.2015e-03,\n         2.4145e-02, -6.7810e-03,  4.3193e-03, -9.8420e-03,  4.2946e-03,\n        -3.4001e-02,  1.9681e-02, -5.0143e-02,  4.6721e-03, -1.0243e-02,\n         2.5696e-02,  2.5677e-02,  1.9892e-02,  1.7967e-02,  3.0659e-02,\n         2.1791e-02,  4.6654e-02, -7.0152e-03,  1.8268e-02, -2.5544e-02,\n         2.0009e-02, -9.4655e-03, -1.7811e-02,  2.9602e-02,  2.8997e-03,\n        -1.4952e-03, -9.9654e-03,  3.2031e-02, -2.6881e-02, -1.4700e-03,\n        -4.5593e-03, -1.1338e-02,  1.3465e-02,  1.8647e-02,  3.7578e-02,\n        -1.5106e-02,  3.4059e-02, -5.7665e-03, -8.1558e-02,  2.2063e-02,\n         1.2213e-02, -6.5718e-04, -4.4406e-02,  3.9488e-03, -4.6688e-03,\n         5.4635e-03,  6.0091e-02, -2.3929e-02, -2.1405e-02,  9.9763e-03,\n         2.0685e-02,  3.9020e-02, -1.0965e-02,  2.6911e-02, -4.3579e-04,\n        -1.3090e-02,  1.7274e-02, -3.1163e-03, -9.2600e-03,  1.4022e-02,\n        -6.7875e-04,  3.3383e-02,  3.1797e-03, -7.2624e-02, -4.0245e-02,\n        -7.3960e-03,  1.8220e-03,  1.2892e-02,  2.1588e-02, -2.1245e-02,\n         6.6080e-03,  1.1453e-02, -5.2910e-03, -3.4239e-03, -3.8178e-02,\n         1.0073e-02, -4.8568e-02, -1.2064e-02,  3.1033e-02, -7.4807e-03,\n         4.6812e-03,  8.5549e-04, -1.5595e-02, -6.6269e-03,  1.7065e-02,\n        -7.2836e-03,  1.9860e-02, -2.9846e-02,  3.1883e-02,  3.1027e-02,\n         2.6315e-02, -3.9592e-02,  7.4055e-03, -1.8968e-02, -6.5877e-03,\n        -2.3735e-03,  1.1277e-02,  2.8674e-02, -2.4042e-02,  8.8102e-03,\n         1.6305e-03,  2.1761e-03, -1.9430e-02,  1.3176e-02,  1.6371e-02,\n        -1.1393e-02,  9.3054e-03,  1.6695e-02,  3.6046e-02, -1.5997e-02,\n         5.8704e-03, -6.3442e-03,  7.4822e-03,  1.4986e-02, -2.5562e-02,\n        -2.2582e-02,  3.9476e-02,  1.1170e-02, -8.8030e-03, -3.6317e-03,\n         1.9700e-02, -2.6428e-02,  3.2573e-03, -7.8790e-04, -2.0253e-02,\n         5.5626e-02,  1.5528e-02, -2.3762e-02,  2.0698e-02, -1.6485e-02,\n         2.6636e-02,  2.0913e-02, -5.1903e-03,  2.6833e-02, -7.5169e-02,\n        -2.0675e-02,  1.6474e-02, -1.5630e-02, -1.9307e-02,  8.8459e-03,\n        -1.3435e-02, -1.0893e-02,  2.4490e-02, -5.5529e-02,  9.5027e-03,\n        -2.1495e-02, -2.3331e-02, -1.1058e-02,  1.4088e-03,  3.0574e-02,\n         1.9587e-02, -1.6454e-02,  1.6400e-02,  2.0025e-02, -7.9974e-02,\n        -5.3161e-02,  2.2492e-02, -4.0501e-02,  1.9811e-02, -1.3800e-02,\n         1.5500e-02, -2.6817e-02,  3.1618e-02,  7.0208e-02,  1.2115e-02,\n         8.5462e-03,  3.2569e-03, -6.5149e-03,  5.7269e-04,  5.7215e-03,\n         1.2468e-02, -1.2838e-02,  5.0595e-03,  1.1441e-02,  2.8161e-02,\n         4.9867e-03, -6.2480e-03, -1.1412e-02, -2.7113e-03,  2.0603e-02,\n        -2.0901e-02, -1.0947e-02, -8.3352e-03, -1.3568e-02,  7.8854e-03,\n         3.1089e-02, -2.5084e-03, -2.0144e-02,  1.1372e-02,  1.2535e-02,\n        -6.6783e-03,  2.0941e-02, -1.0867e-02, -3.0452e-02,  3.7372e-02,\n        -1.8344e-02,  1.4053e-02, -3.9700e-02,  1.2862e-02,  1.3314e-03,\n         1.6941e-02,  3.3995e-02,  1.4878e-03, -7.5845e-03, -1.2363e-02,\n         3.5461e-03,  1.1331e-02, -3.3280e-02, -2.1071e-02,  7.7722e-03,\n        -3.0055e-03,  1.0837e-02, -2.7664e-02,  1.6052e-02,  8.3437e-03,\n        -2.3247e-02, -1.9109e-03, -2.5256e-03, -3.2315e-02, -1.7712e-02,\n         1.4877e-02, -2.2091e-03, -3.8271e-03, -7.7975e-03,  2.0651e-02,\n        -5.8186e-03,  1.1344e-02, -8.7831e-03, -5.0037e-02, -6.2444e-03,\n        -1.7972e-02, -3.5386e-02,  2.3158e-02, -8.3040e-03, -2.8092e-03,\n        -2.8807e-03,  1.0920e-02,  6.7379e-03,  2.9991e-02, -1.0605e-02,\n         7.1341e-03,  1.6258e-02,  1.0589e-02,  1.0662e-02, -2.8346e-03,\n         1.8843e-02,  8.8679e-03,  4.9969e-03,  1.2956e-02,  7.1300e-03,\n         2.1672e-02, -4.2474e-04,  1.5483e-03, -3.6167e-03,  4.1395e-03,\n         1.0872e-02, -1.5972e-02, -1.9861e-03,  1.0547e-02, -1.9846e-04,\n        -1.3616e-02, -3.8203e-02,  2.5938e-03, -4.1131e-02, -1.4700e-02,\n        -5.9516e-02,  1.0883e-02,  2.0863e-03, -1.2418e-02,  2.5647e-02,\n        -4.6071e-06,  1.2153e-02,  4.6008e-03, -2.9608e-03,  5.9074e-04,\n        -2.7776e-02,  1.1034e-03,  1.0020e-02, -5.5500e-03, -1.2561e-02,\n         3.6402e-03,  1.9298e-02,  1.0005e-02,  7.2212e-03,  3.0687e-02,\n         3.9395e-03, -1.0360e-02, -1.5357e-02,  1.1455e-02,  3.1281e-03,\n        -4.6923e-02,  2.5320e-02, -2.3042e-02, -1.1396e-02,  1.9229e-04,\n        -2.3274e-02,  1.7475e-02, -2.0032e-02,  6.7570e-03, -2.0166e-02,\n        -1.4327e-02, -3.5414e-02,  1.1922e-02, -7.0488e-03,  9.3528e-03,\n         2.6386e-02, -2.3849e-02, -4.0436e-02,  8.1401e-03,  8.0442e-03,\n         1.3884e-02,  1.3633e-02, -3.2131e-02,  1.1315e-02, -2.5761e-02,\n         1.3171e-02, -1.4911e-02,  1.5835e-02,  1.3933e-02, -1.5586e-02,\n         2.7392e-03,  1.3112e-02,  9.9083e-03, -2.3326e-03, -8.6873e-04,\n         1.0280e-02, -9.1481e-03,  2.3267e-02, -7.0494e-04, -2.5615e-03,\n         3.4622e-02, -1.3005e-02,  1.6936e-02, -7.7853e-03,  3.8981e-03,\n        -1.7965e-02, -7.5781e-03,  1.4724e-02, -3.2101e-02,  1.3328e-03,\n         2.8313e-03,  3.6617e-02, -4.5399e-02, -1.9580e-02, -1.2964e-02,\n        -2.0127e-02, -9.2166e-03,  3.6274e-02,  3.2222e-02,  1.5137e-02,\n         5.5583e-02, -2.7585e-02, -4.2199e-03,  2.1378e-02, -4.6858e-02,\n         1.4901e-02, -6.7604e-03, -1.4126e-02, -2.4446e-02, -2.7160e-02,\n        -3.7297e-02, -1.8781e-02,  3.9964e-02, -9.0569e-03, -1.2340e-02,\n        -1.6683e-02, -5.3954e-03,  1.2370e-02, -1.7175e-02,  1.7346e-02,\n         2.7414e-02, -1.5188e-02, -1.5477e-02, -1.6274e-01,  5.7006e-03,\n         3.4489e-02,  1.2587e-02, -3.1556e-03, -1.3249e-02,  1.0066e-02,\n        -1.7941e-02,  1.2355e-02,  2.8834e-02, -5.9926e-03, -3.4528e-02,\n        -1.0967e-02,  7.2966e-03,  2.1129e-04,  3.9717e-02, -3.4769e-02,\n        -5.3892e-03,  1.5771e-02,  6.7886e-03,  8.9231e-03,  1.6861e-02,\n         7.8304e-03,  9.3717e-03, -2.2886e-02, -6.8705e-03,  1.8815e-02,\n        -7.8668e-03, -5.3651e-03, -6.3996e-03,  1.3167e-02,  9.1540e-03,\n         4.2709e-03, -3.5334e-02,  2.5007e-02,  7.1316e-03,  4.4322e-02,\n         9.3333e-03,  8.7682e-03, -2.4643e-03, -1.1923e-02,  1.7582e-02,\n         2.3745e-03,  8.2075e-03, -4.0503e-03, -2.2440e-02,  2.1457e-04,\n        -8.0670e-03,  1.4730e-02, -1.3338e-03,  1.7442e-03, -1.6567e-02,\n        -1.8960e-02,  9.9549e-03, -4.1571e-02, -2.3103e-03,  3.0346e-02,\n         3.5036e-04, -9.3455e-03,  8.4965e-03,  2.6898e-02, -1.4490e-02,\n        -3.5875e-02,  2.9421e-02, -2.0395e-03, -3.8398e-03,  4.5688e-03,\n        -2.3263e-02, -1.5443e-02, -1.6684e-02,  1.9446e-02, -1.5944e-03,\n        -1.0828e-02,  1.7730e-02,  4.9617e-04,  5.0250e-03, -1.9821e-02,\n        -6.6361e-03, -7.6752e-03], device='cuda:0')",
+        "exp_avg_sq": "tensor([0.0055, 0.0074, 0.0043, 0.0030, 0.0051, 0.0037, 0.0042, 0.0037, 0.0061,\n        0.0046, 0.0037, 0.0047, 0.0061, 0.0042, 0.0053, 0.0049, 0.0013, 0.0028,\n        0.0036, 0.0035, 0.0038, 0.0039, 0.0042, 0.0053, 0.0059, 0.0034, 0.0033,\n        0.0041, 0.0070, 0.0023, 0.0038, 0.0068, 0.0042, 0.0046, 0.0032, 0.0036,\n        0.0027, 0.0046, 0.0045, 0.0041, 0.0061, 0.0037, 0.0052, 0.0071, 0.0049,\n        0.0057, 0.0065, 0.0044, 0.0028, 0.0049, 0.0033, 0.0039, 0.0054, 0.0029,\n        0.0069, 0.0045, 0.0024, 0.0051, 0.0036, 0.0037, 0.0033, 0.0067, 0.0055,\n        0.0030, 0.0025, 0.0036, 0.0038, 0.0053, 0.0089, 0.0054, 0.0085, 0.0047,\n        0.0087, 0.0037, 0.0049, 0.0085, 0.0053, 0.0047, 0.0033, 0.0079, 0.0072,\n        0.0047, 0.0068, 0.0048, 0.0035, 0.0034, 0.0011, 0.0090, 0.0061, 0.0054,\n        0.0039, 0.0038, 0.0028, 0.0047, 0.0068, 0.0045, 0.0074, 0.0061, 0.0044,\n        0.0056, 0.0041, 0.0046, 0.0045, 0.0023, 0.0037, 0.0042, 0.0085, 0.0033,\n        0.0063, 0.0055, 0.0024, 0.0067, 0.0040, 0.0039, 0.0031, 0.0038, 0.0057,\n        0.0071, 0.0030, 0.0048, 0.0043, 0.0028, 0.0055, 0.0057, 0.0076, 0.0048,\n        0.0052, 0.0037, 0.0059, 0.0032, 0.0050, 0.0044, 0.0032, 0.0027, 0.0054,\n        0.0031, 0.0047, 0.0027, 0.0065, 0.0084, 0.0049, 0.0031, 0.0038, 0.0065,\n        0.0058, 0.0043, 0.0044, 0.0045, 0.0052, 0.0056, 0.0040, 0.0043, 0.0030,\n        0.0034, 0.0033, 0.0034, 0.0048, 0.0051, 0.0027, 0.0065, 0.0032, 0.0041,\n        0.0048, 0.0046, 0.0055, 0.0047, 0.0024, 0.0034, 0.0051, 0.0049, 0.0042,\n        0.0043, 0.0025, 0.0044, 0.0051, 0.0031, 0.0038, 0.0046, 0.0034, 0.0051,\n        0.0055, 0.0083, 0.0026, 0.0047, 0.0060, 0.0054, 0.0017, 0.0029, 0.0052,\n        0.0034, 0.0023, 0.0026, 0.0062, 0.0049, 0.0060, 0.0042, 0.0044, 0.0041,\n        0.0051, 0.0088, 0.0025, 0.0039, 0.0031, 0.0069, 0.0043, 0.0028, 0.0025,\n        0.0034, 0.0037, 0.0062, 0.0030, 0.0062, 0.0053, 0.0065, 0.0037, 0.0033,\n        0.0040, 0.0039, 0.0036, 0.0051, 0.0033, 0.0051, 0.0069, 0.0075, 0.0029,\n        0.0052, 0.0041, 0.0063, 0.0040, 0.0073, 0.0045, 0.0038, 0.0059, 0.0031,\n        0.0044, 0.0066, 0.0048, 0.0053, 0.0044, 0.0089, 0.0066, 0.0039, 0.0051,\n        0.0038, 0.0030, 0.0057, 0.0041, 0.0046, 0.0034, 0.0056, 0.0030, 0.0035,\n        0.0020, 0.0078, 0.0055, 0.0052, 0.0069, 0.0047, 0.0051, 0.0047, 0.0046,\n        0.0069, 0.0028, 0.0026, 0.0024, 0.0034, 0.0036, 0.0060, 0.0028, 0.0048,\n        0.0025, 0.0025, 0.0053, 0.0053, 0.0047, 0.0054, 0.0018, 0.0034, 0.0081,\n        0.0054, 0.0053, 0.0052, 0.0049, 0.0036, 0.0052, 0.0056, 0.0030, 0.0046,\n        0.0060, 0.0026, 0.0041, 0.0065, 0.0043, 0.0049, 0.0072, 0.0054, 0.0076,\n        0.0052, 0.0092, 0.0070, 0.0038, 0.0039, 0.0038, 0.0037, 0.0039, 0.0040,\n        0.0050, 0.0037, 0.0066, 0.0046, 0.0047, 0.0061, 0.0034, 0.0046, 0.0024,\n        0.0049, 0.0034, 0.0045, 0.0049, 0.0030, 0.0052, 0.0063, 0.0051, 0.0038,\n        0.0053, 0.0046, 0.0030, 0.0046, 0.0038, 0.0036, 0.0065, 0.0050, 0.0038,\n        0.0037, 0.0026, 0.0031, 0.0043, 0.0034, 0.0056, 0.0053, 0.0035, 0.0038,\n        0.0035, 0.0046, 0.0054, 0.0048, 0.0038, 0.0043, 0.0032, 0.0050, 0.0040,\n        0.0076, 0.0039, 0.0046, 0.0033, 0.0049, 0.0070, 0.0044, 0.0026, 0.0053,\n        0.0052, 0.0055, 0.0050, 0.0027, 0.0039, 0.0052, 0.0035, 0.0085, 0.0034,\n        0.0035, 0.0033, 0.0103, 0.0063, 0.0050, 0.0047, 0.0035, 0.0051, 0.0033,\n        0.0051, 0.0037, 0.0038, 0.0047, 0.0040, 0.0048, 0.0064, 0.0040, 0.0026,\n        0.0055, 0.0063, 0.0021, 0.0059, 0.0083, 0.0033, 0.0043, 0.0033, 0.0048,\n        0.0027, 0.0034, 0.0059, 0.0027, 0.0029, 0.0052, 0.0055, 0.0071, 0.0057,\n        0.0090, 0.0046, 0.0032, 0.0059, 0.0026, 0.0049, 0.0060, 0.0067, 0.0032,\n        0.0061, 0.0035, 0.0041, 0.0032, 0.0083, 0.0034, 0.0049, 0.0044, 0.0058,\n        0.0057, 0.0089, 0.0047, 0.0069, 0.0059, 0.0033, 0.0033, 0.0052, 0.0035,\n        0.0034, 0.0142, 0.0032, 0.0056, 0.0058, 0.0056, 0.0051, 0.0059, 0.0066,\n        0.0065, 0.0042, 0.0033, 0.0031, 0.0039, 0.0061, 0.0028, 0.0037, 0.0042,\n        0.0020, 0.0073, 0.0052, 0.0044, 0.0035, 0.0071, 0.0055, 0.0026, 0.0048,\n        0.0040, 0.0054, 0.0068, 0.0034, 0.0071, 0.0014, 0.0042, 0.0037, 0.0053,\n        0.0040, 0.0055, 0.0048, 0.0030, 0.0064, 0.0049, 0.0033, 0.0033, 0.0037,\n        0.0033, 0.0054, 0.0039, 0.0044, 0.0052, 0.0038, 0.0043, 0.0042, 0.0038,\n        0.0042, 0.0039, 0.0088, 0.0048, 0.0007, 0.0054, 0.0042, 0.0042, 0.0058,\n        0.0045, 0.0038, 0.0059, 0.0048, 0.0043, 0.0040, 0.0045, 0.0067, 0.0041,\n        0.0048, 0.0061, 0.0040, 0.0028, 0.0044, 0.0042, 0.0050, 0.0040],\n       device='cuda:0')"
+      },
+      "2": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([ 2.5797e-03, -3.3814e-03,  3.0553e-03, -4.8659e-04,  8.4865e-04,\n         1.7258e-03, -5.2457e-03,  7.1769e-04,  2.8211e-03,  1.4553e-03,\n        -4.4405e-03,  1.9009e-03,  5.7904e-04, -6.7648e-04,  1.7537e-03,\n         2.2422e-04,  1.8196e-02,  1.0818e-03,  8.3466e-04, -4.6780e-03,\n        -4.4582e-03, -6.4097e-03,  5.3906e-03,  1.2437e-03,  1.4201e-03,\n        -3.0965e-03,  8.2905e-04, -1.0077e-02, -5.3708e-04,  1.8073e-03,\n         1.9708e-03,  7.0104e-03, -2.1256e-03,  8.1101e-04,  4.0544e-04,\n        -4.0117e-03, -2.2346e-03,  4.8665e-03,  2.0623e-03, -2.9910e-03,\n         1.4717e-03, -1.7680e-03,  7.5650e-04, -4.4887e-04, -1.2551e-03,\n         5.8311e-03,  3.6320e-04,  2.9798e-03,  3.3360e-03,  8.3932e-04,\n        -7.3543e-03,  9.0433e-03, -1.0963e-03,  3.6825e-03,  3.9170e-03,\n         5.8936e-03, -2.1031e-03,  5.2774e-03,  2.7684e-04, -7.6586e-04,\n         4.3258e-03, -9.1538e-04,  6.8676e-04, -1.9899e-03,  1.0481e-03,\n        -5.3377e-03,  4.6545e-03, -5.7779e-03, -1.2246e-03, -3.5930e-04,\n         1.8451e-03,  3.5630e-03,  9.4508e-04,  2.7300e-03,  4.9102e-03,\n         1.7281e-03,  5.8309e-03,  8.3630e-05,  3.1348e-03, -2.3868e-03,\n         2.5132e-03, -1.2820e-03, -1.9902e-03,  2.9845e-03,  7.0080e-04,\n        -1.1548e-03, -6.1439e-03,  3.0292e-03, -3.3293e-03,  2.4139e-05,\n        -1.6863e-03, -1.5370e-03,  3.3356e-03,  2.6120e-03,  5.1559e-03,\n        -8.9724e-04,  6.0912e-03, -1.1774e-03, -9.1221e-03,  2.6078e-03,\n         1.9146e-03,  1.0183e-04, -5.5212e-03,  1.7406e-03, -7.2280e-04,\n         2.4157e-03,  6.2427e-03, -4.3606e-03, -2.3678e-03,  7.6681e-04,\n         7.9451e-03,  3.2661e-03, -1.3919e-03,  4.4425e-03, -8.4783e-06,\n        -8.7055e-04,  7.2784e-04, -5.0674e-04, -2.9537e-03,  2.5050e-03,\n        -8.8498e-04,  7.5670e-03,  1.1001e-03, -7.2467e-03, -3.7582e-03,\n         5.2763e-04,  1.6729e-03,  3.1186e-03,  1.9995e-03, -6.2119e-03,\n         1.1065e-04,  8.3647e-04, -1.0844e-03, -1.3158e-03, -7.9446e-03,\n         1.2190e-03, -6.4509e-03, -4.3690e-03,  3.7780e-03, -1.4501e-03,\n         4.4605e-04, -1.4421e-04, -1.6538e-03, -4.6284e-04,  2.1583e-03,\n        -1.7942e-03,  9.9423e-04, -3.6512e-03,  4.0913e-03,  3.2393e-03,\n         3.5763e-03, -7.7524e-03,  1.5525e-03, -3.2299e-03, -1.4703e-04,\n        -5.9182e-05,  2.5233e-03,  4.4227e-03, -7.5911e-03,  3.8186e-04,\n         3.1270e-04, -2.0654e-04, -2.5328e-03,  1.6714e-03,  4.1853e-03,\n        -1.1523e-03,  3.3213e-03,  2.7456e-03,  2.9781e-03, -2.2021e-03,\n         2.5656e-03, -1.3444e-03,  4.0176e-03,  2.0946e-03, -3.2350e-03,\n        -7.7431e-03,  6.9712e-03,  2.0539e-03, -1.9416e-03, -9.9022e-04,\n         2.6571e-03, -2.8907e-03,  1.3489e-03, -5.9392e-04, -5.0839e-03,\n         9.1166e-03,  6.7417e-03, -7.3623e-03,  2.4900e-03, -4.2655e-03,\n         1.0480e-02,  6.8648e-03, -1.9055e-03,  2.8732e-03, -8.5434e-03,\n        -3.0694e-03,  2.0111e-03, -3.9273e-03, -1.7570e-03,  4.1386e-04,\n        -3.9600e-03, -4.1857e-03,  3.4499e-03, -8.6752e-03,  1.1484e-03,\n        -6.3472e-03, -7.7676e-03, -1.1460e-03,  4.3473e-04,  3.5277e-03,\n         3.8496e-03, -1.8652e-03,  4.4663e-03,  1.6806e-03, -1.4994e-02,\n        -1.0339e-02,  3.9079e-03, -6.1276e-03,  8.5781e-04, -1.5473e-03,\n         1.9379e-03, -2.7702e-03,  2.2611e-03,  9.4913e-03,  3.8446e-03,\n         1.0249e-03,  5.4911e-04, -9.7949e-04, -3.0946e-04,  1.0067e-03,\n         1.6643e-03, -2.2037e-03,  1.0704e-04,  1.3573e-03,  4.1211e-03,\n        -1.9724e-05, -1.4774e-04, -2.9566e-03, -1.5479e-03,  1.0522e-03,\n        -1.8503e-03, -3.1776e-03, -9.5384e-04, -2.4167e-03,  1.6889e-03,\n         4.4344e-03, -8.7977e-04, -2.8082e-03,  2.0169e-03,  1.3686e-03,\n        -1.7260e-03,  3.9149e-03, -5.3617e-03, -2.6511e-03,  4.5934e-03,\n        -2.5222e-03, -8.6372e-04, -6.6812e-03,  2.1650e-03,  1.2320e-03,\n         1.6753e-03,  3.3796e-03,  2.2866e-03, -1.4567e-03, -6.6092e-03,\n         7.7998e-04,  2.3545e-03, -4.3264e-03, -4.6893e-03,  9.0112e-04,\n        -1.9495e-03,  1.1600e-03, -1.6197e-03,  1.2812e-03,  2.7142e-04,\n        -2.1607e-03, -1.9067e-03, -6.3638e-04, -5.2843e-03, -1.1780e-03,\n         9.3220e-04,  8.5335e-04, -9.4985e-04, -8.9068e-04,  2.8323e-03,\n        -1.1532e-03,  4.8930e-03, -1.1576e-03, -6.0586e-03, -2.4100e-03,\n        -2.8446e-03, -3.2128e-03,  2.8963e-03, -9.4006e-04,  2.2745e-03,\n        -1.2111e-03,  6.4743e-04,  1.7391e-03,  3.0069e-03, -1.5925e-03,\n         1.6493e-03,  1.2351e-03,  1.0096e-03,  2.5747e-03, -1.4471e-03,\n         3.0768e-03,  2.1878e-03,  4.0370e-04,  2.5063e-03,  1.4776e-03,\n         3.2077e-03,  1.0391e-03,  4.4717e-04, -7.9502e-04,  1.7201e-04,\n         2.2257e-03, -2.3748e-03, -8.0869e-04,  1.6408e-03, -7.6032e-04,\n        -5.8402e-04, -8.8262e-03, -3.7939e-04, -6.8342e-03, -4.9823e-04,\n        -7.8423e-03,  1.1002e-03,  4.8165e-04, -1.4593e-03,  3.8792e-03,\n        -2.9298e-04,  1.3732e-03,  1.3019e-03, -5.9476e-04, -5.8806e-04,\n        -6.8732e-03, -6.5804e-04,  1.7609e-03, -1.0404e-03, -1.1750e-03,\n        -1.7199e-04,  3.9768e-03,  1.7005e-03,  1.6230e-03,  3.9329e-03,\n         5.0513e-04, -1.4404e-03, -2.2788e-03,  2.5884e-03,  7.1543e-04,\n        -8.4716e-03,  3.5636e-03, -3.3957e-03, -1.5896e-03, -2.0890e-04,\n        -3.2824e-03,  8.8820e-04, -2.3560e-03,  1.3940e-03, -2.0064e-03,\n        -2.3570e-03, -2.5264e-03,  1.8627e-03, -1.4167e-03,  1.8815e-03,\n         5.4344e-03, -4.5462e-03, -2.3670e-03,  7.2561e-04,  4.0388e-03,\n         2.3688e-03,  5.0867e-04, -3.6029e-03,  2.0150e-03, -3.8132e-03,\n         2.0181e-03, -1.4927e-03,  2.3594e-03,  1.0554e-03, -2.9342e-03,\n        -1.1649e-04,  2.0772e-03,  3.4173e-03,  1.2992e-04, -7.8788e-04,\n         1.9356e-03, -1.3653e-03,  2.1818e-03, -5.2551e-04, -3.2516e-03,\n         5.8402e-03, -3.6830e-04,  4.3402e-03, -1.1397e-03,  1.1419e-03,\n        -3.6876e-03, -1.9010e-03,  4.1173e-03, -3.8112e-03,  7.7829e-04,\n         1.9187e-03,  4.7110e-03, -3.3995e-03, -3.2822e-03, -1.0035e-03,\n        -2.1042e-03, -1.3309e-03,  6.8720e-03,  3.4705e-03,  4.3246e-03,\n         8.8855e-03, -2.1834e-03,  6.4634e-05,  5.9686e-03, -2.2851e-03,\n         2.8330e-03, -1.6221e-03, -3.8866e-03, -2.5062e-03, -8.3753e-03,\n        -5.6303e-03, -3.7971e-03,  4.1722e-03, -2.2608e-03, -1.1552e-03,\n        -2.1976e-03, -1.3545e-03,  1.5978e-03, -3.8724e-03,  4.9018e-03,\n         2.3121e-03, -2.5956e-03, -4.0751e-03, -5.3382e-03,  1.8889e-03,\n         5.0081e-03,  9.2713e-04, -2.0956e-03, -2.4197e-03,  6.7707e-04,\n        -1.8208e-03,  1.5737e-03,  4.8269e-03, -9.8244e-04, -7.3700e-03,\n        -1.5821e-03,  3.3851e-04, -5.0511e-05,  8.3870e-03, -8.1216e-03,\n        -3.4790e-03,  1.8361e-03,  6.8586e-04,  1.2406e-03,  4.4521e-03,\n         9.9590e-04,  1.6832e-03, -7.0484e-03, -2.0344e-03,  4.7944e-03,\n         1.2277e-04, -8.3653e-04, -7.7410e-04,  1.9681e-03,  8.4979e-03,\n         8.7261e-04, -3.4650e-03,  2.0717e-03,  7.7992e-04,  4.8661e-03,\n         5.4598e-04,  3.3359e-03,  8.6542e-04, -1.4784e-03,  3.4084e-03,\n        -2.0170e-04,  2.3598e-03,  6.6625e-04, -6.7647e-04, -7.5519e-04,\n        -1.3545e-03,  1.1575e-03, -3.5629e-05,  5.4802e-04, -6.6094e-04,\n        -4.3212e-03,  1.2341e-03, -6.8009e-03, -2.0429e-04,  4.5444e-03,\n        -1.0796e-01, -1.5517e-03,  1.3062e-03,  4.0417e-03, -4.5133e-03,\n        -5.9013e-03,  6.3230e-03, -2.3557e-04, -6.1693e-04,  1.4088e-04,\n        -3.6728e-03, -1.9465e-03, -1.4361e-03,  3.8910e-03,  3.7007e-04,\n        -2.3836e-03,  2.9955e-03, -5.7067e-04,  6.9021e-04, -2.2165e-03,\n        -1.8113e-03, -2.2169e-03], device='cuda:0')",
+        "exp_avg_sq": "tensor([9.3534e-05, 5.2081e-05, 8.7579e-05, 1.1775e-04, 3.7490e-05, 6.6398e-05,\n        6.5181e-05, 7.9934e-05, 3.7579e-05, 7.8748e-05, 7.5650e-05, 1.0794e-04,\n        4.4289e-05, 7.4519e-05, 3.4879e-05, 4.8113e-05, 9.5417e-04, 9.0605e-05,\n        7.7610e-05, 1.0413e-04, 1.3113e-04, 1.2828e-04, 1.0292e-04, 7.1092e-05,\n        6.9161e-05, 1.2308e-04, 9.0564e-05, 9.4796e-05, 4.1548e-05, 2.0471e-04,\n        7.7138e-05, 6.2262e-05, 1.2604e-04, 4.2145e-05, 1.7723e-04, 1.4397e-04,\n        1.1216e-04, 1.3823e-04, 4.4817e-05, 8.0767e-05, 6.6478e-05, 6.6847e-05,\n        3.5678e-05, 4.5877e-05, 6.6082e-05, 4.4817e-05, 1.1796e-04, 7.8770e-05,\n        1.3425e-04, 6.6750e-05, 1.2493e-04, 1.4512e-04, 4.9036e-05, 1.1805e-04,\n        6.7659e-05, 9.4551e-05, 1.6406e-04, 8.5441e-05, 7.8109e-05, 9.5035e-05,\n        1.1370e-04, 3.8469e-05, 7.5884e-05, 1.5717e-04, 2.3834e-04, 9.1314e-05,\n        9.6325e-05, 6.9950e-05, 4.2466e-05, 7.6099e-05, 4.4453e-05, 6.3488e-05,\n        5.3895e-05, 5.9271e-05, 8.4895e-05, 7.5968e-05, 7.6019e-05, 6.8775e-05,\n        8.9169e-05, 8.8133e-05, 6.3438e-05, 4.1043e-05, 7.7734e-05, 7.5004e-05,\n        6.9372e-05, 1.1152e-04, 8.2787e-04, 6.8318e-05, 5.1235e-05, 1.0371e-04,\n        1.0030e-04, 1.1779e-04, 9.8979e-05, 4.6600e-05, 6.2193e-05, 5.2905e-05,\n        8.7848e-05, 6.7115e-05, 6.4299e-05, 4.7252e-05, 4.0719e-05, 4.4532e-05,\n        4.7058e-05, 1.3056e-04, 1.3770e-04, 1.0543e-04, 5.1060e-05, 1.2723e-04,\n        6.5365e-05, 4.7783e-05, 2.3576e-04, 4.5003e-05, 5.9560e-05, 4.8028e-05,\n        1.8568e-04, 5.8620e-05, 4.8536e-05, 5.8821e-05, 1.0382e-04, 6.7783e-05,\n        6.2843e-05, 1.0138e-04, 6.9408e-05, 7.1709e-05, 7.1197e-05, 5.2420e-05,\n        4.4202e-05, 7.8958e-05, 5.7452e-05, 1.5871e-04, 4.5656e-05, 7.2952e-05,\n        1.5122e-04, 1.0611e-04, 8.3228e-05, 1.0585e-04, 5.5973e-05, 1.4356e-04,\n        5.9684e-05, 7.0793e-05, 1.0179e-04, 1.0461e-04, 7.0807e-05, 7.7465e-05,\n        6.7427e-05, 9.3235e-05, 7.1464e-05, 7.9589e-05, 6.2760e-05, 5.9923e-05,\n        3.6313e-05, 1.0735e-04, 6.5802e-05, 1.3033e-04, 6.5644e-05, 1.2888e-04,\n        7.8531e-05, 6.5305e-05, 2.2830e-04, 4.1322e-05, 1.1254e-04, 7.4478e-05,\n        9.1465e-05, 7.5960e-05, 8.3753e-05, 4.9294e-05, 1.8695e-04, 1.0751e-04,\n        4.3824e-05, 6.4196e-05, 1.3429e-04, 7.3125e-05, 2.0548e-04, 6.8109e-05,\n        6.2985e-05, 9.3164e-05, 6.6411e-05, 9.9192e-05, 6.1117e-05, 4.4956e-05,\n        5.6639e-05, 7.1771e-05, 2.5785e-04, 7.7366e-05, 6.7656e-05, 9.6274e-05,\n        2.5144e-04, 2.0722e-04, 6.0485e-05, 1.1283e-04, 1.6085e-04, 1.8561e-04,\n        8.7354e-05, 7.0174e-05, 8.4883e-05, 6.2343e-05, 4.4061e-05, 6.2873e-05,\n        4.3974e-05, 6.0602e-05, 1.3165e-04, 9.8687e-05, 4.0801e-05, 6.0715e-05,\n        6.2841e-05, 1.2023e-04, 1.6530e-04, 1.1902e-04, 7.9409e-05, 7.2037e-05,\n        7.9170e-05, 6.8876e-05, 6.8475e-05, 7.6226e-05, 1.2973e-04, 8.1130e-05,\n        1.0440e-04, 4.9289e-05, 7.3087e-05, 7.9991e-05, 8.6720e-05, 4.4938e-05,\n        4.1638e-05, 9.1885e-05, 1.4029e-04, 7.0141e-05, 9.1576e-05, 4.6775e-05,\n        1.6586e-04, 1.0802e-04, 4.0638e-05, 7.1225e-05, 4.4447e-05, 7.4944e-05,\n        7.9197e-05, 7.7527e-05, 8.6782e-05, 7.9764e-05, 1.0183e-04, 7.2638e-05,\n        4.6090e-05, 9.5125e-05, 8.1266e-05, 6.5972e-05, 1.5332e-04, 5.0552e-05,\n        5.7805e-05, 3.6570e-05, 1.0344e-04, 7.0497e-05, 7.6239e-05, 7.1408e-05,\n        1.7463e-04, 6.2077e-05, 7.0871e-05, 4.3119e-05, 1.4044e-04, 7.3603e-05,\n        9.3906e-05, 8.3015e-05, 5.5512e-05, 5.6514e-05, 1.0255e-04, 1.3032e-04,\n        3.1997e-04, 8.7664e-05, 7.3213e-05, 6.4461e-05, 1.0469e-04, 6.0136e-05,\n        1.2104e-04, 7.7671e-05, 4.3034e-05, 4.5255e-05, 4.7679e-05, 1.4406e-04,\n        3.1668e-04, 8.9568e-05, 8.3074e-05, 6.2992e-05, 2.9778e-05, 7.9814e-05,\n        7.2739e-05, 9.1559e-05, 7.9475e-05, 5.2932e-05, 3.2108e-04, 4.6038e-05,\n        7.2401e-05, 8.5028e-05, 1.0851e-04, 6.7335e-05, 8.3705e-05, 5.8019e-05,\n        7.5169e-05, 7.3187e-05, 4.5433e-05, 1.1059e-04, 5.3843e-05, 5.9505e-05,\n        1.2189e-04, 7.3994e-05, 5.6143e-05, 1.7383e-04, 6.0140e-05, 9.8114e-05,\n        6.4706e-05, 7.4265e-05, 8.5531e-05, 6.7183e-05, 6.1264e-05, 4.6472e-05,\n        7.5889e-05, 6.2791e-05, 1.4004e-04, 8.6030e-05, 4.9368e-05, 8.9827e-05,\n        4.9411e-05, 1.3412e-04, 4.3134e-05, 1.2779e-04, 6.3193e-05, 8.3734e-05,\n        5.3592e-05, 5.8036e-05, 8.3066e-05, 6.6027e-05, 4.3307e-05, 6.1910e-05,\n        6.2722e-05, 1.4815e-04, 5.6265e-05, 7.7929e-05, 2.2987e-04, 1.2527e-04,\n        7.1161e-05, 1.2677e-04, 8.0058e-05, 5.5377e-05, 9.8939e-05, 9.8796e-05,\n        5.7770e-05, 7.1164e-05, 6.8153e-05, 4.9240e-05, 7.8473e-05, 4.7828e-05,\n        1.3589e-04, 4.4736e-05, 1.0090e-04, 5.4520e-05, 3.5023e-05, 5.5291e-05,\n        9.3725e-05, 5.2801e-05, 3.2232e-05, 4.8235e-05, 1.9287e-04, 6.1198e-05,\n        6.4328e-05, 3.3087e-05, 5.6195e-05, 8.8625e-05, 4.1878e-05, 8.3448e-05,\n        5.8431e-05, 6.6324e-05, 8.1913e-05, 9.5008e-05, 1.2768e-04, 7.9377e-05,\n        6.0684e-05, 8.1082e-05, 5.7340e-05, 1.0374e-04, 4.8783e-05, 6.5684e-05,\n        5.5993e-05, 1.1619e-04, 4.0398e-05, 5.7264e-05, 9.2368e-05, 9.7118e-05,\n        5.1494e-05, 4.9772e-05, 8.5448e-05, 3.9619e-05, 3.2178e-05, 3.3220e-04,\n        7.8344e-05, 4.8256e-05, 1.0854e-04, 7.6845e-05, 1.2860e-04, 9.9543e-05,\n        1.3782e-04, 1.4173e-04, 6.0130e-05, 1.3464e-04, 1.3802e-04, 6.4300e-05,\n        4.7140e-05, 5.1204e-05, 4.8817e-05, 6.0277e-05, 6.9688e-05, 6.6136e-05,\n        5.7686e-05, 1.8014e-04, 6.8162e-05, 4.3598e-05, 7.8203e-05, 2.0398e-04,\n        4.6620e-05, 1.5014e-04, 5.3713e-05, 6.5638e-05, 5.8817e-05, 1.5822e-04,\n        1.0967e-04, 7.1899e-05, 5.6858e-05, 5.9848e-05, 5.1107e-05, 6.9600e-05,\n        5.9563e-05, 5.9783e-05, 1.0691e-04, 1.2999e-04, 4.5842e-05, 1.9972e-04,\n        1.1130e-04, 5.8034e-05, 9.0860e-05, 9.2545e-05, 4.2140e-05, 7.1882e-05,\n        6.7124e-05, 4.5057e-05, 7.7587e-05, 7.7103e-05, 5.4419e-05, 1.3975e-04,\n        8.5213e-05, 4.2136e-05, 5.4884e-05, 1.1120e-04, 1.0581e-04, 1.1924e-04,\n        2.2009e-04, 8.1455e-05, 4.3195e-05, 1.2521e-04, 1.0591e-04, 6.5561e-05,\n        3.4314e-05, 2.2634e-04, 8.2888e-05, 7.2406e-05, 1.0405e-04, 5.6693e-05,\n        1.2943e-04, 4.7538e-05, 3.7359e-04, 9.6639e-05, 6.1040e-05, 4.5744e-05,\n        9.4919e-05, 3.8998e-05, 5.5567e-05, 1.0872e-04, 5.4458e-05, 8.8826e-05,\n        9.4942e-05, 1.1226e-04, 7.8136e-05, 9.4123e-05, 2.7194e-05, 7.0199e-05,\n        5.3198e-05, 2.7440e-05, 1.0667e-04, 9.0163e-05, 5.3747e-05, 1.2569e-04,\n        5.4848e-05, 8.6609e-05, 7.6437e-05, 6.3086e-05, 3.8596e-02, 4.7234e-05,\n        7.5285e-05, 6.9735e-05, 7.4087e-05, 1.2637e-04, 1.0606e-04, 7.2766e-05,\n        5.2593e-05, 9.2583e-05, 6.0584e-05, 7.3959e-05, 6.5191e-05, 7.6121e-05,\n        7.0820e-05, 1.0032e-04, 1.0296e-04, 8.7977e-05, 9.5822e-05, 1.1168e-04,\n        9.7474e-05, 7.8855e-05], device='cuda:0')"
+      },
+      "3": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([ 3.8952e-03, -4.5050e-03,  3.3686e-03, -1.2955e-03,  1.4151e-03,\n         1.4463e-03, -4.7583e-03,  1.2517e-03,  2.9852e-03,  2.5372e-03,\n        -4.9426e-03,  2.6334e-03,  5.7251e-05, -8.1991e-04,  1.5303e-03,\n         9.4863e-04,  6.7579e-03,  7.8777e-04, -1.4215e-03, -1.7853e-03,\n        -2.1686e-03, -5.9553e-03,  4.6379e-03,  7.2355e-04,  2.3988e-03,\n        -3.0314e-03, -4.4116e-04, -5.1131e-03, -7.0414e-04,  1.7755e-03,\n         1.6376e-03,  8.8029e-03, -1.6398e-03,  1.2540e-03,  6.6523e-04,\n        -3.3510e-03, -9.7379e-04,  4.0501e-03,  2.7495e-03, -2.1269e-03,\n         1.2958e-03, -2.3034e-03,  1.0309e-03, -9.6372e-04, -9.3579e-04,\n         6.8909e-03,  8.7533e-04,  2.7002e-03,  2.1199e-03,  6.4311e-04,\n        -3.4483e-03,  8.2710e-03, -2.6438e-03,  3.5739e-03,  4.4153e-03,\n         4.6829e-03, -4.4914e-05,  5.6990e-03,  3.5162e-04,  3.8782e-04,\n         4.6787e-03, -7.5045e-04,  1.2159e-03, -1.6735e-03,  8.1080e-04,\n        -5.9700e-03,  3.9532e-03, -6.4655e-03,  2.5421e-04, -1.1731e-03,\n         3.1420e-03,  3.4559e-03,  2.3272e-03,  2.6877e-03,  4.7084e-03,\n         2.5773e-03,  7.1018e-03, -1.1365e-03,  3.2770e-03, -3.1603e-03,\n         3.3430e-03, -1.6630e-03, -2.5677e-03,  4.2299e-03,  7.3000e-04,\n        -7.0014e-04, -2.9378e-03,  3.1971e-03, -3.5376e-03, -3.0454e-04,\n        -6.7089e-04, -1.3966e-03,  2.7905e-03,  2.5077e-03,  6.0016e-03,\n        -1.9116e-03,  5.0675e-03, -9.3987e-04, -1.1224e-02,  2.8105e-03,\n         1.9126e-03, -8.7026e-05, -5.6501e-03,  1.1775e-03, -8.7144e-04,\n         7.2139e-04,  7.9853e-03, -3.9899e-03, -3.1926e-03,  1.2823e-03,\n         4.7541e-03,  5.0309e-03, -1.9141e-03,  4.2647e-03,  2.2053e-04,\n        -1.7861e-03,  1.6340e-03, -1.2048e-04, -2.2014e-03,  2.1449e-03,\n        -1.9473e-04,  5.8081e-03,  7.3048e-04, -1.0283e-02, -5.3062e-03,\n        -6.3674e-04,  8.6553e-04,  2.5146e-03,  3.2712e-03, -4.5801e-03,\n         1.0315e-03,  1.4187e-03, -1.2515e-03, -8.5546e-04, -5.2848e-03,\n         2.0511e-03, -6.9212e-03, -2.1491e-03,  5.3276e-03, -1.0095e-03,\n         4.3892e-04, -9.8881e-05, -1.6663e-03, -5.3300e-04,  2.3004e-03,\n        -1.2771e-03,  2.0184e-03, -5.5092e-03,  4.7244e-03,  4.6795e-03,\n         3.5509e-03, -6.2852e-03,  1.4859e-03, -4.3219e-03, -8.5340e-05,\n        -4.9808e-04,  1.6404e-03,  4.6351e-03, -5.7846e-03,  1.1594e-03,\n         1.0102e-04,  3.7960e-04, -3.2925e-03,  1.6834e-03,  2.8801e-03,\n        -1.4512e-03,  1.9714e-03,  2.6050e-03,  4.8240e-03, -2.5881e-03,\n         1.2944e-03, -1.3944e-03,  2.1690e-03,  2.0877e-03, -3.3894e-03,\n        -4.4894e-03,  6.4366e-03,  2.0231e-03, -1.7648e-03, -5.9508e-04,\n         2.9853e-03, -3.7564e-03,  6.2361e-04,  5.7978e-05, -3.2855e-03,\n         8.0948e-03,  3.8235e-03, -5.3936e-03,  2.2583e-03, -2.8611e-03,\n         5.9956e-03,  4.0346e-03, -1.4107e-03,  3.7592e-03, -1.0832e-02,\n        -3.3580e-03,  2.2872e-03, -3.2459e-03, -2.4541e-03,  1.6004e-03,\n        -3.6087e-03, -2.1214e-03,  3.7118e-03, -6.6466e-03,  1.5329e-03,\n        -4.2209e-03, -5.8850e-03, -1.6751e-03,  4.6243e-04,  3.6608e-03,\n         3.3828e-03, -2.1083e-03,  2.0112e-03,  3.1592e-03, -1.3673e-02,\n        -8.7569e-03,  4.2278e-03, -5.9717e-03,  4.0565e-03, -1.7651e-03,\n         2.8577e-03, -3.1889e-03,  3.7271e-03,  9.3227e-03,  2.7167e-03,\n         1.3422e-03,  4.3974e-04, -8.5102e-04, -1.0581e-04,  1.0947e-03,\n         1.8829e-03, -2.0908e-03,  7.2278e-04,  1.7325e-03,  4.8608e-03,\n         7.6701e-05, -9.8171e-04, -2.3452e-03, -1.4928e-04,  2.1193e-03,\n        -2.4678e-03, -1.9260e-03, -1.3081e-03, -2.4267e-03,  1.6899e-03,\n         4.5309e-03, -1.2568e-04, -3.1195e-03,  2.1411e-03,  1.5926e-03,\n        -1.4611e-03,  3.9234e-03, -2.8925e-03, -5.0136e-03,  5.4693e-03,\n        -2.7708e-03,  2.0058e-03, -7.3481e-03,  1.5997e-03,  9.0944e-04,\n         2.2591e-03,  4.7433e-03,  1.3036e-03, -1.5587e-03, -3.4030e-03,\n         4.5099e-04,  2.1168e-03, -4.5584e-03, -3.5693e-03,  1.1453e-03,\n        -3.2446e-04,  1.4992e-03, -3.3115e-03,  2.0432e-03,  1.3700e-03,\n        -3.3539e-03, -8.9838e-04, -8.8088e-04, -4.5651e-03, -2.1369e-03,\n         1.4949e-03, -7.4546e-04, -7.8435e-04, -1.1129e-03,  3.2665e-03,\n        -1.4047e-03,  2.9993e-03, -1.7921e-03, -6.9232e-03, -6.9246e-04,\n        -2.8294e-03, -4.4985e-03,  4.1200e-03, -8.2095e-04, -9.9116e-05,\n        -3.3664e-04,  1.4158e-03,  1.3417e-03,  4.1731e-03, -1.3329e-03,\n         2.1331e-03,  2.6603e-03,  1.3275e-03,  1.7912e-03, -7.8859e-04,\n         3.3742e-03,  1.5810e-03,  2.9862e-04,  1.6168e-03,  1.3023e-03,\n         3.5993e-03,  5.7664e-05,  2.9951e-04, -2.2677e-04,  3.9571e-04,\n         8.1708e-04, -2.8095e-03, -5.8858e-04,  1.5378e-03, -2.5739e-04,\n        -1.4165e-03, -6.7982e-03, -1.1885e-04, -6.7117e-03, -1.5221e-03,\n        -9.1270e-03,  1.5554e-03,  2.7269e-04, -1.4548e-03,  4.3686e-03,\n         2.5415e-04,  2.3150e-03,  1.0176e-03, -4.3034e-04, -2.8216e-04,\n        -5.4079e-03, -7.0299e-04,  1.9824e-03, -8.6843e-04, -1.4289e-03,\n         4.3305e-04,  3.5364e-03,  1.6810e-03,  1.0776e-03,  4.2726e-03,\n         7.1858e-04, -1.6763e-03, -1.5440e-03,  2.0539e-03,  3.7411e-04,\n        -9.3839e-03,  3.4082e-03, -3.4259e-03, -2.0100e-03,  4.7492e-04,\n        -3.3385e-03,  1.9260e-03, -2.7886e-03,  1.9537e-03, -3.1827e-03,\n        -1.8783e-03, -3.7541e-03,  1.8200e-03, -1.3720e-03,  1.3670e-03,\n         4.7278e-03, -3.4608e-03, -4.6856e-03,  8.3006e-04,  2.0356e-03,\n         2.5413e-03,  1.3200e-03, -4.7444e-03,  2.3437e-03, -3.7137e-03,\n         2.2891e-03, -2.0898e-03,  2.2255e-03,  1.8834e-03, -2.6077e-03,\n         2.9853e-05,  2.0375e-03,  1.8496e-03, -4.4447e-04, -3.7637e-04,\n         1.3387e-03, -1.6411e-03,  3.1724e-03, -2.6364e-04, -9.3110e-04,\n         5.6093e-03, -1.5935e-03,  2.9264e-03, -1.4463e-03,  8.4222e-04,\n        -3.4978e-03, -1.5978e-03,  3.2323e-03, -4.6861e-03, -9.7839e-05,\n         2.7957e-04,  6.1117e-03, -6.3786e-03, -2.6717e-03, -1.6021e-03,\n        -2.1478e-03, -1.5544e-03,  5.8230e-03,  4.6057e-03,  3.0332e-03,\n         8.3149e-03, -4.5162e-03, -3.2785e-04,  4.6230e-03, -4.6572e-03,\n         3.1800e-03, -5.9412e-04, -2.3277e-03, -3.2297e-03, -5.3461e-03,\n        -5.9741e-03, -2.7955e-03,  5.1513e-03, -1.3157e-03, -1.1130e-03,\n        -2.4742e-03, -6.2188e-04,  2.1274e-03, -3.1207e-03,  3.4336e-03,\n         3.5235e-03, -2.8090e-03, -3.2575e-03, -1.5983e-02,  1.2347e-03,\n         5.6146e-03,  1.5307e-03, -7.5069e-04, -2.6315e-03,  1.4185e-03,\n        -2.5394e-03,  1.5071e-03,  5.5175e-03, -1.3205e-03, -6.5679e-03,\n        -1.1976e-03,  8.2539e-04,  6.3877e-04,  7.4866e-03, -6.8664e-03,\n        -1.7564e-03,  1.8901e-03,  8.8146e-04,  1.7445e-03,  3.3627e-03,\n         9.3994e-04,  1.6042e-03, -4.9658e-03, -2.2933e-03,  3.5600e-03,\n        -1.4322e-03, -5.1243e-04, -1.3976e-03,  1.5368e-03,  3.2724e-03,\n         5.0387e-04, -4.9651e-03,  3.3767e-03,  8.3414e-04,  6.0335e-03,\n         1.3685e-03,  2.1346e-03, -7.4662e-05, -2.2401e-03,  3.2065e-03,\n         1.1653e-03,  1.6231e-03, -7.8231e-04, -1.9919e-03, -1.0642e-04,\n        -1.0256e-03,  1.6487e-03, -2.6958e-04,  3.9595e-04, -1.8813e-03,\n        -3.1388e-03,  1.5435e-03, -6.6359e-03,  1.4578e-04,  4.6597e-03,\n        -5.5814e-03, -1.4072e-03,  1.6976e-03,  4.1522e-03, -3.3935e-03,\n        -5.6314e-03,  5.5278e-03, -5.3896e-04, -3.8658e-04,  1.0674e-03,\n        -4.4977e-03, -2.1185e-03, -2.2955e-03,  3.4403e-03,  6.7157e-05,\n        -1.6017e-03,  2.6573e-03, -2.3024e-04,  3.6171e-04, -2.2036e-03,\n        -1.8412e-03, -1.1724e-03], device='cuda:0')",
+        "exp_avg_sq": "tensor([1.0226e-04, 9.5872e-05, 6.0648e-05, 7.9654e-05, 5.0524e-05, 6.6973e-05,\n        6.8471e-05, 6.8566e-05, 6.3886e-05, 8.8335e-05, 6.7515e-05, 9.9880e-05,\n        5.8344e-05, 7.1379e-05, 5.1134e-05, 6.1063e-05, 1.2513e-04, 6.9882e-05,\n        6.8014e-05, 7.5321e-05, 8.1574e-05, 9.3176e-05, 8.1562e-05, 7.1760e-05,\n        9.6404e-05, 9.0608e-05, 7.2865e-05, 6.9661e-05, 6.0765e-05, 8.5385e-05,\n        7.0276e-05, 9.3393e-05, 1.0309e-04, 6.4943e-05, 9.0773e-05, 9.4972e-05,\n        7.1667e-05, 9.1020e-05, 5.2736e-05, 7.1932e-05, 7.7926e-05, 6.4304e-05,\n        6.0936e-05, 7.4240e-05, 7.9894e-05, 6.3644e-05, 1.2462e-04, 8.0605e-05,\n        7.0121e-05, 7.4467e-05, 7.1571e-05, 9.6243e-05, 6.7134e-05, 7.5973e-05,\n        9.9941e-05, 9.0950e-05, 7.9527e-05, 8.4598e-05, 7.1917e-05, 7.5104e-05,\n        8.8285e-05, 5.8515e-05, 9.4971e-05, 8.9861e-05, 8.7893e-05, 7.5787e-05,\n        7.9506e-05, 8.0095e-05, 8.1752e-05, 9.0099e-05, 8.9016e-05, 6.3561e-05,\n        9.6012e-05, 6.0710e-05, 9.2865e-05, 1.1773e-04, 8.4632e-05, 7.5973e-05,\n        7.3998e-05, 1.1287e-04, 1.0048e-04, 6.4853e-05, 1.0600e-04, 8.1949e-05,\n        6.5489e-05, 7.9015e-05, 9.6872e-05, 1.0325e-04, 6.0927e-05, 1.0132e-04,\n        9.5073e-05, 8.8626e-05, 6.9521e-05, 5.9851e-05, 8.8287e-05, 7.1213e-05,\n        1.1330e-04, 9.0229e-05, 7.3622e-05, 6.6657e-05, 5.1270e-05, 6.1973e-05,\n        5.8645e-05, 6.7765e-05, 9.3269e-05, 9.2882e-05, 9.2530e-05, 7.7720e-05,\n        9.1586e-05, 6.8289e-05, 9.0872e-05, 6.8703e-05, 7.4267e-05, 6.2981e-05,\n        1.0472e-04, 6.3881e-05, 5.5625e-05, 9.7625e-05, 7.3338e-05, 6.4065e-05,\n        6.1516e-05, 6.0991e-05, 8.3844e-05, 8.9363e-05, 1.0287e-04, 6.2397e-05,\n        6.2724e-05, 6.6554e-05, 7.1243e-05, 9.0916e-05, 6.8823e-05, 7.7818e-05,\n        8.8947e-05, 5.9778e-05, 8.2051e-05, 7.3195e-05, 7.2101e-05, 7.9064e-05,\n        8.6894e-05, 8.6149e-05, 1.0037e-04, 7.8463e-05, 5.7536e-05, 9.9137e-05,\n        7.6975e-05, 8.4262e-05, 7.9278e-05, 8.3809e-05, 7.5214e-05, 8.5006e-05,\n        4.6147e-05, 8.4582e-05, 5.2558e-05, 8.9936e-05, 5.7913e-05, 9.4974e-05,\n        8.0749e-05, 7.6560e-05, 9.9720e-05, 6.1732e-05, 7.5119e-05, 6.9414e-05,\n        1.0225e-04, 7.0636e-05, 9.4689e-05, 6.2350e-05, 8.1075e-05, 7.5130e-05,\n        6.4571e-05, 8.2642e-05, 1.0857e-04, 7.1447e-05, 8.7742e-05, 6.3170e-05,\n        8.1316e-05, 6.1477e-05, 6.6704e-05, 8.6427e-05, 5.6795e-05, 5.8434e-05,\n        7.2105e-05, 1.1565e-04, 9.2252e-05, 7.8355e-05, 9.3501e-05, 8.4195e-05,\n        7.4385e-05, 9.8639e-05, 6.7922e-05, 7.7217e-05, 7.2878e-05, 8.2008e-05,\n        1.0885e-04, 7.6865e-05, 8.6024e-05, 6.3777e-05, 5.7957e-05, 7.4695e-05,\n        6.3721e-05, 1.0341e-04, 7.9330e-05, 8.6606e-05, 4.4167e-05, 8.4026e-05,\n        7.0570e-05, 6.9532e-05, 7.9395e-05, 7.6493e-05, 7.4033e-05, 9.0855e-05,\n        6.4168e-05, 9.5819e-05, 8.3112e-05, 9.3385e-05, 9.2383e-05, 6.7443e-05,\n        8.4251e-05, 5.2427e-05, 6.6575e-05, 7.5303e-05, 7.5404e-05, 6.6914e-05,\n        7.4998e-05, 1.0974e-04, 8.1290e-05, 8.5425e-05, 6.4894e-05, 6.6105e-05,\n        1.1012e-04, 1.2450e-04, 6.4975e-05, 6.5283e-05, 7.1307e-05, 6.4585e-05,\n        7.8013e-05, 1.1023e-04, 8.5546e-05, 1.0156e-04, 9.6419e-05, 1.0992e-04,\n        6.8793e-05, 7.8676e-05, 9.2537e-05, 7.2813e-05, 8.0951e-05, 7.7447e-05,\n        6.2905e-05, 4.7405e-05, 8.1435e-05, 8.6134e-05, 6.4573e-05, 6.4277e-05,\n        6.7504e-05, 1.0898e-04, 8.1345e-05, 5.9156e-05, 1.5431e-04, 9.0643e-05,\n        8.8715e-05, 8.7856e-05, 6.9906e-05, 9.0095e-05, 6.3042e-05, 6.8681e-05,\n        9.9719e-05, 6.9366e-05, 6.6496e-05, 7.6886e-05, 6.7989e-05, 6.8355e-05,\n        6.7529e-05, 5.2884e-05, 5.8253e-05, 5.9654e-05, 6.9907e-05, 1.1602e-04,\n        7.6332e-05, 8.0069e-05, 1.1695e-04, 9.0939e-05, 5.0891e-05, 9.7106e-05,\n        8.2772e-05, 7.2002e-05, 8.8952e-05, 6.5962e-05, 1.0206e-04, 5.9860e-05,\n        7.5653e-05, 6.1120e-05, 9.6379e-05, 9.3336e-05, 8.8618e-05, 6.8093e-05,\n        1.0991e-04, 8.0660e-05, 7.9534e-05, 1.1611e-04, 8.6071e-05, 1.0257e-04,\n        8.9021e-05, 7.5626e-05, 6.2310e-05, 1.1045e-04, 6.5077e-05, 8.3483e-05,\n        7.1596e-05, 6.3488e-05, 9.6715e-05, 7.3082e-05, 6.8115e-05, 6.5073e-05,\n        6.1120e-05, 6.1207e-05, 6.5513e-05, 8.4483e-05, 5.7027e-05, 7.6576e-05,\n        6.6347e-05, 7.7977e-05, 5.5341e-05, 1.1301e-04, 9.0276e-05, 7.5203e-05,\n        6.9007e-05, 7.8409e-05, 6.0793e-05, 8.3727e-05, 5.0282e-05, 6.0686e-05,\n        8.1741e-05, 1.1023e-04, 5.8451e-05, 6.2856e-05, 9.4704e-05, 7.7485e-05,\n        6.7695e-05, 8.5719e-05, 9.5210e-05, 7.7369e-05, 7.5747e-05, 8.4593e-05,\n        5.7598e-05, 7.1198e-05, 7.5705e-05, 6.7421e-05, 6.5667e-05, 6.1567e-05,\n        8.1948e-05, 4.2496e-05, 8.6122e-05, 8.0704e-05, 4.6315e-05, 6.6944e-05,\n        7.6653e-05, 6.7168e-05, 6.4184e-05, 6.2249e-05, 9.4317e-05, 6.3705e-05,\n        7.0715e-05, 5.1065e-05, 6.5800e-05, 6.2255e-05, 4.8393e-05, 9.4064e-05,\n        5.3379e-05, 1.1402e-04, 6.0975e-05, 6.9631e-05, 7.8219e-05, 1.1760e-04,\n        8.8647e-05, 8.3666e-05, 6.4279e-05, 7.9398e-05, 6.7879e-05, 6.4008e-05,\n        7.7656e-05, 7.4064e-05, 5.1194e-05, 6.9647e-05, 8.5499e-05, 9.0750e-05,\n        7.7803e-05, 5.0122e-05, 6.2523e-05, 6.1458e-05, 6.5829e-05, 1.0522e-04,\n        9.0705e-05, 7.4638e-05, 7.8718e-05, 6.4506e-05, 8.5827e-05, 9.4571e-05,\n        7.6017e-05, 8.3075e-05, 8.9544e-05, 7.5614e-05, 7.5364e-05, 8.4915e-05,\n        7.3320e-05, 7.8103e-05, 6.1017e-05, 8.8997e-05, 7.2262e-05, 5.6697e-05,\n        8.0657e-05, 8.7817e-05, 7.1980e-05, 6.2690e-05, 9.2671e-05, 1.0964e-04,\n        6.5436e-05, 9.1867e-05, 5.4442e-05, 6.0076e-05, 1.0443e-04, 9.4694e-05,\n        9.0859e-05, 6.5475e-05, 7.7995e-05, 7.2718e-05, 9.9340e-05, 7.4778e-05,\n        8.1433e-05, 6.6613e-05, 7.4962e-05, 8.5345e-05, 6.6328e-05, 9.9380e-05,\n        7.9887e-05, 1.4267e-04, 6.5538e-05, 1.0480e-04, 6.2213e-05, 9.8979e-05,\n        8.5380e-05, 6.5950e-05, 9.3384e-05, 1.0610e-04, 6.6893e-05, 8.2503e-05,\n        6.8071e-05, 5.5594e-05, 7.6909e-05, 6.7313e-05, 8.2332e-05, 9.4519e-05,\n        8.2998e-05, 8.4469e-05, 5.7382e-05, 9.9397e-05, 8.5323e-05, 9.5525e-05,\n        6.4003e-05, 9.1201e-05, 8.0505e-05, 7.5474e-05, 1.0061e-04, 9.2399e-05,\n        8.5650e-05, 7.5300e-05, 7.6542e-05, 8.1315e-05, 6.3995e-05, 6.7666e-05,\n        6.5320e-05, 6.2039e-05, 6.5023e-05, 7.4188e-05, 7.9473e-05, 9.6191e-05,\n        6.7424e-05, 7.7789e-05, 7.1906e-05, 7.1455e-05, 4.9418e-05, 6.7479e-05,\n        6.6581e-05, 5.3691e-05, 7.9586e-05, 7.0861e-05, 5.9944e-05, 8.1818e-05,\n        5.9490e-05, 7.2205e-05, 9.9444e-05, 7.6687e-05, 1.1669e-04, 7.1087e-05,\n        7.5541e-05, 6.8311e-05, 9.8890e-05, 9.3615e-05, 8.2114e-05, 8.9609e-05,\n        6.4294e-05, 8.0709e-05, 7.3411e-05, 6.7581e-05, 9.5257e-05, 7.1498e-05,\n        8.0444e-05, 1.0883e-04, 8.5270e-05, 6.2424e-05, 7.6163e-05, 7.9111e-05,\n        9.1720e-05, 6.7865e-05], device='cuda:0')"
+      },
+      "4": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[ 2.0956e-04, -4.4868e-04, -6.7421e-05,  ...,  2.6711e-04,\n         -2.7648e-05, -1.9781e-04],\n        [-1.2116e-04,  1.0628e-03,  7.7454e-05,  ..., -4.4967e-04,\n         -2.0475e-06,  8.4231e-04],\n        [ 8.6658e-04,  3.8975e-04, -1.5171e-04,  ..., -6.4941e-04,\n          6.6473e-05,  3.0153e-04],\n        ...,\n        [-2.1107e-05, -5.4272e-04, -1.0084e-04,  ..., -8.3872e-04,\n         -8.3126e-04, -7.3044e-06],\n        [-2.8526e-04,  8.9263e-05, -9.8878e-05,  ..., -4.3640e-04,\n          4.1936e-04, -3.0780e-04],\n        [-5.0747e-05,  8.8016e-04,  1.5652e-04,  ...,  3.8422e-04,\n         -4.5725e-04, -2.6752e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[7.6681e-07, 8.7230e-07, 4.4272e-07,  ..., 6.2036e-07, 7.1349e-07,\n         8.3666e-07],\n        [1.3779e-06, 2.1535e-06, 1.1529e-06,  ..., 1.5845e-06, 1.7256e-06,\n         1.7896e-06],\n        [1.3142e-06, 9.7473e-07, 1.0733e-06,  ..., 1.6766e-06, 1.2405e-06,\n         1.4313e-06],\n        ...,\n        [1.4296e-06, 1.9809e-06, 1.3485e-06,  ..., 1.6085e-06, 1.9970e-06,\n         1.5226e-06],\n        [1.3013e-06, 1.5834e-06, 8.6765e-07,  ..., 1.9837e-06, 1.5881e-06,\n         1.4278e-06],\n        [1.2881e-06, 1.2104e-06, 1.0649e-06,  ..., 2.2814e-06, 1.2303e-06,\n         1.6723e-06]], device='cuda:0')"
+      },
+      "5": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([[ 1.1705e-05, -1.1827e-06, -9.0512e-06,  ...,  3.9217e-05,\n         -9.3789e-05, -6.3116e-05],\n        [-1.8033e-05,  3.7945e-04,  3.8273e-05,  ..., -1.1815e-04,\n         -2.6072e-05,  4.5533e-04],\n        [ 4.1122e-04,  1.2079e-04, -1.4540e-04,  ..., -3.3914e-04,\n          1.8452e-04,  2.0942e-04],\n        ...,\n        [ 4.4994e-05, -5.0683e-04,  5.5247e-06,  ..., -4.1807e-04,\n         -5.1996e-04, -1.1124e-04],\n        [ 1.9861e-04, -3.2420e-04,  1.0195e-05,  ...,  1.1968e-04,\n          1.9824e-04, -5.4688e-05],\n        [-3.4761e-05,  1.2445e-04,  8.2007e-05,  ..., -2.4124e-04,\n         -3.5979e-05,  2.1654e-04]], device='cuda:0')",
+        "exp_avg_sq": "tensor([[2.0447e-07, 1.5624e-07, 1.8918e-07,  ..., 2.4198e-07, 2.0178e-07,\n         2.1589e-07],\n        [4.0951e-07, 6.6674e-07, 3.7600e-07,  ..., 3.5996e-07, 4.4268e-07,\n         5.0101e-07],\n        [4.1429e-07, 3.4825e-07, 3.7328e-07,  ..., 5.9834e-07, 5.2854e-07,\n         4.1828e-07],\n        ...,\n        [4.1243e-07, 7.8513e-07, 2.9527e-07,  ..., 7.0980e-07, 6.7169e-07,\n         4.4539e-07],\n        [4.7271e-07, 4.9985e-07, 2.5180e-07,  ..., 6.2623e-07, 4.8837e-07,\n         4.3894e-07],\n        [4.5230e-07, 2.6609e-07, 6.8972e-07,  ..., 4.6541e-07, 4.0036e-07,\n         5.4594e-07]], device='cuda:0')"
+      },
+      "6": {
+        "step": "tensor(3756.)",
+        "exp_avg": "tensor([-0.0008,  0.0008], device='cuda:0')",
+        "exp_avg_sq": "tensor([1.2195e-05, 1.2195e-05], device='cuda:0')"
+      }
+    },
+    "param_groups": [
+      {
+        "lr": 0.0007940987335200904,
+        "name": "shared",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          0,
+          1,
+          2,
+          3
+        ]
+      },
+      {
+        "lr": 0.0007940987335200904,
+        "name": "scale_256",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          4
+        ]
+      },
+      {
+        "lr": 0.0007940987335200904,
+        "name": "scale_512",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.001,
+        "params": [
+          5
+        ]
+      },
+      {
+        "lr": 0.00039715242044697206,
+        "name": "fusion",
+        "betas": [
+          0.9,
+          0.999
+        ],
+        "eps": 1e-08,
+        "weight_decay": 1e-05,
+        "amsgrad": false,
+        "maximize": false,
+        "foreach": null,
+        "capturable": false,
+        "differentiable": false,
+        "fused": null,
+        "decoupled_weight_decay": true,
+        "initial_lr": 0.0005,
+        "params": [
+          6
+        ]
+      }
+    ]
+  },
+  "scheduler_state_dict": {
+    "T_0": 10,
+    "T_i": 10,
+    "T_mult": 2,
+    "eta_min": 1e-06,
+    "T_cur": 3,
+    "base_lrs": [
+      0.001,
+      0.001,
+      0.001,
+      0.0005
+    ],
+    "last_epoch": 3,
+    "_step_count": 0,
+    "_is_initial": false,
+    "_get_lr_called_within_step": false,
+    "_last_lr": [
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.0007940987335200904,
+      0.00039715242044697206
+    ]
+  },
+  "metrics": {
+    "best_val_acc": 70.142,
+    "best_epoch": 2,
+    "scale_accuracies": {
+      "256": 69.54,
+      "512": 70.028
+    },
+    "training_history": {
+      "epochs": [
+        1,
+        2,
+        3
+      ],
+      "train_loss": [
+        5.603825211905824,
+        4.145695094292918,
+        3.767625377201044
+      ],
+      "train_acc": [
+        63.39829233815732,
+        69.48282308239285,
+        70.34266414917025
+      ],
+      "val_acc": [
+        68.056,
+        69.624,
+        70.142
+      ],
+      "scale_accs": {
+        "256": [
+          66.924,
+          68.828,
+          69.54
+        ],
+        "512": [
+          67.882,
+          69.572,
+          70.028
+        ]
+      },
+      "lr": [
+        0.0009755527298894294,
+        0.0009046039886902864,
+        0.0007940987335200904
+      ]
+    }
+  },
+  "train_config": {
+    "name": "david_training",
+    "run_id": "20251012_135249",
+    "dataset_name": "AbstractPhil/imagenet-clip-features-orderly",
+    "model_variant": "clip_vit_laion_b32",
+    "num_classes": 1000,
+    "preset": "small_fast",
+    "custom_config_path": null,
+    "num_classes_override": null,
+    "use_belly_override": null,
+    "belly_expand_override": null,
+    "progressive_training_override": false,
+    "scale_warmup_epochs_override": null,
+    "num_epochs": 10,
+    "batch_size": 1024,
+    "learning_rate": 0.001,
+    "weight_decay": 1e-05,
+    "warmup_epochs": 3,
+    "use_rose_loss": true,
+    "rose_initial_weight": 0.1,
+    "rose_max_weight": 0.5,
+    "rose_weight_schedule": "adaptive",
+    "use_cayley_loss": false,
+    "cayley_weight": 0.001,
+    "scale_loss_balance": null,
+    "use_mixed_precision": false,
+    "gradient_clip": 10.0,
+    "scheduler_type": "cosine_restarts",
+    "min_lr": 1e-06,
+    "freeze_strategy": "never",
+    "freeze_threshold": 90.0,
+    "unfreeze_on_plateau": true,
+    "patience": 10,
+    "track_gradients": true,
+    "gradient_scale_threshold": 1e-05,
+    "gradient_scale_multiplier": 10.0,
+    "log_interval": 50,
+    "val_interval": 1,
+    "save_interval": 5,
+    "log_fusion_weights": true,
+    "log_loss_components": true,
+    "save_format": "safetensors",
+    "hf_repo": "AbstractPhil/gated-david",
+    "upload_to_hub": true,
+    "base_dir": "./david_training",
+    "num_workers": 10,
+    "pin_memory": true,
+    "prefetch_factor": 4,
+    "persistent_workers": true
+  }
+}

weights/David-fully_shared-weighted_sum/20251012_135249/training_history.json

@@ -1,32 +1,39 @@
 {
   "epochs": [
     1,
-    2
+    2,
+    3
   ],
   "train_loss": [
     5.603825211905824,
-    4.145695094292918
+    4.145695094292918,
+    3.767625377201044
   ],
   "train_acc": [
     63.39829233815732,
-    69.48282308239285
+    69.48282308239285,
+    70.34266414917025
   ],
   "val_acc": [
     68.056,
-    69.624
+    69.624,
+    70.142
   ],
   "scale_accs": {
     "256": [
       66.924,
-      68.828
+      68.828,
+      69.54
     ],
     "512": [
       67.882,
-      69.572
+      69.572,
+      70.028
     ]
   },
   "lr": [
     0.0009755527298894294,
-    0.0009046039886902864
+    0.0009046039886902864,
+    0.0007940987335200904
   ]
 }