Upload folder using huggingface_hub

experiments/2026-04-13-120000-noise-robust-v2/config.json

@@ -0,0 +1 @@
+{"in_channels": 24, "channels": [32, 64, 128, 256], "context_len": 8, "model_class": "FlowWarpAttnUNet"}

experiments/2026-04-13-120000-noise-robust-v2/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ecf0b11a223818197d6e19840408c3f2a9f425de2643394fceb40dfc9013a99
+size 15223306

experiments/2026-04-13-120000-noise-robust-v2/predict.py

@@ -0,0 +1,59 @@
+"""Inference for AR curriculum model + TTA."""
+import json
+import numpy as np
+import torch
+import sys
+sys.path.insert(0, "/home/coder/code")
+from flow_warp_attn_model import FlowWarpAttnUNet
+
+
+def load_model(model_dir: str):
+    with open(f"{model_dir}/config.json") as f:
+        config = json.load(f)
+    model = FlowWarpAttnUNet(in_channels=config["in_channels"], channels=config["channels"])
+    sd = torch.load(f"{model_dir}/model.pt", map_location="cpu", weights_only=True)
+    sd = {k: v.float() for k, v in sd.items()}
+    model.load_state_dict(sd)
+    model.eval()
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = model.to(device)
+    return {"model": model, "device": device, "context_len": config["context_len"]}
+
+
+def _prepare_input(context_frames, context_len):
+    N = len(context_frames)
+    if N >= context_len:
+        frames = context_frames[-context_len:]
+    else:
+        pad = np.repeat(context_frames[:1], context_len - N, axis=0)
+        frames = np.concatenate([pad, context_frames], axis=0)
+    frames_f = frames.astype(np.float32) / 255.0
+    frames_f = np.transpose(frames_f, (0, 3, 1, 2))
+    context = frames_f.reshape(1, -1, 64, 64)
+    last_frame = frames_f[-1:]
+    return context, last_frame
+
+
+def predict_next_frame(model_dict, context_frames: np.ndarray) -> np.ndarray:
+    model = model_dict["model"]
+    device = model_dict["device"]
+    context_len = model_dict["context_len"]
+
+    ctx, last = _prepare_input(context_frames, context_len)
+    with torch.no_grad():
+        ctx_t = torch.from_numpy(ctx).to(device)
+        last_t = torch.from_numpy(last).to(device)
+        pred1, _ = model(ctx_t, last_t)
+
+    flipped_frames = context_frames[:, :, ::-1, :].copy()
+    ctx_f, last_f = _prepare_input(flipped_frames, context_len)
+    with torch.no_grad():
+        ctx_ft = torch.from_numpy(ctx_f).to(device)
+        last_ft = torch.from_numpy(last_f).to(device)
+        pred2, _ = model(ctx_ft, last_ft)
+        pred2 = pred2.flip(-1)
+
+    pred = (pred1 + pred2) / 2.0
+    pred_np = pred[0].cpu().numpy()
+    pred_np = np.transpose(pred_np, (1, 2, 0))
+    return (pred_np * 255.0).clip(0, 255).astype(np.uint8)

experiments/2026-04-13-120000-noise-robust-v2/train.log

@@ -0,0 +1,260 @@
+[12:47:47] === Stage A: Generating diverse error patterns ===
+[12:47:47] Generating errors from attn-big...
+[12:47:53]   attn-big batch 0/347
+[12:47:55]   attn-big batch 50/347
+[12:47:57] Generating errors from optim-v1...
+[12:48:02]   optim-v1 batch 0/347
+[12:48:04]   optim-v1 batch 50/347
+[12:48:07]   Step 1: 3000 errors, mean abs = 0.0074
+[12:48:07]   Step 2: 3000 errors, mean abs = 0.0106
+[12:48:07]   Step 3: 3000 errors, mean abs = 0.0130
+[12:48:07]   Step 4: 3000 errors, mean abs = 0.0155
+[12:48:08]   Step 5: 3000 errors, mean abs = 0.0181
+[12:48:08]   Step 6: 3000 errors, mean abs = 0.0204
+[12:48:08]   Step 7: 3000 errors, mean abs = 0.0224
+[12:48:08]   Step 8: 3000 errors, mean abs = 0.0245
+[12:48:08] Error pattern generation complete.
+[12:48:08] Model: FlowWarpAttnUNet (noise-robust v2), 7,596,742 params
+[12:48:08] === Phase 1: 4-step AR with heavy noise (80 epochs) ===
+[12:48:15] Train: 11098, Val: 1364
+[12:49:46] P1 Ep 1/80 | Train: 0.064112 | Val: 0.104390 | LR: 2.00e-05 | TF: 0.20 | NP: 0.70
+[12:49:46]   -> Saved P1 (val=0.104390)
+[12:51:14] P1 Ep 2/80 | Train: 0.062642 | Val: 0.103521 | LR: 2.00e-05 | TF: 0.20 | NP: 0.69
+[12:51:14]   -> Saved P1 (val=0.103521)
+[12:52:39] P1 Ep 3/80 | Train: 0.061868 | Val: 0.104480 | LR: 1.99e-05 | TF: 0.19 | NP: 0.68
+[12:54:07] P1 Ep 4/80 | Train: 0.060773 | Val: 0.104548 | LR: 1.99e-05 | TF: 0.19 | NP: 0.67
+[12:55:35] P1 Ep 5/80 | Train: 0.060747 | Val: 0.104922 | LR: 1.98e-05 | TF: 0.18 | NP: 0.66
+[12:57:03] P1 Ep 6/80 | Train: 0.060469 | Val: 0.104017 | LR: 1.97e-05 | TF: 0.18 | NP: 0.66
+[12:58:30] P1 Ep 7/80 | Train: 0.061054 | Val: 0.103998 | LR: 1.96e-05 | TF: 0.17 | NP: 0.65
+[12:59:58] P1 Ep 8/80 | Train: 0.060158 | Val: 0.105022 | LR: 1.95e-05 | TF: 0.17 | NP: 0.64
+[13:01:21] P1 Ep 9/80 | Train: 0.059244 | Val: 0.105245 | LR: 1.94e-05 | TF: 0.16 | NP: 0.63
+[13:02:48] P1 Ep 10/80 | Train: 0.059408 | Val: 0.105614 | LR: 1.93e-05 | TF: 0.16 | NP: 0.62
+[13:04:12] P1 Ep 11/80 | Train: 0.058884 | Val: 0.105173 | LR: 1.91e-05 | TF: 0.15 | NP: 0.61
+[13:05:36] P1 Ep 12/80 | Train: 0.059124 | Val: 0.105281 | LR: 1.90e-05 | TF: 0.14 | NP: 0.60
+[13:06:58] P1 Ep 13/80 | Train: 0.058952 | Val: 0.104825 | LR: 1.88e-05 | TF: 0.14 | NP: 0.59
+[13:08:22] P1 Ep 14/80 | Train: 0.059013 | Val: 0.105102 | LR: 1.86e-05 | TF: 0.14 | NP: 0.59
+[13:09:43] P1 Ep 15/80 | Train: 0.058649 | Val: 0.105226 | LR: 1.84e-05 | TF: 0.13 | NP: 0.58
+[13:11:03] P1 Ep 16/80 | Train: 0.058174 | Val: 0.105889 | LR: 1.82e-05 | TF: 0.12 | NP: 0.57
+[13:12:23] P1 Ep 17/80 | Train: 0.057945 | Val: 0.105069 | LR: 1.80e-05 | TF: 0.12 | NP: 0.56
+[13:13:40] P1 Ep 18/80 | Train: 0.057601 | Val: 0.105332 | LR: 1.77e-05 | TF: 0.11 | NP: 0.55
+[13:14:59] P1 Ep 19/80 | Train: 0.058257 | Val: 0.105534 | LR: 1.75e-05 | TF: 0.11 | NP: 0.54
+[13:16:20] P1 Ep 20/80 | Train: 0.057245 | Val: 0.105607 | LR: 1.72e-05 | TF: 0.11 | NP: 0.53
+[13:17:35] P1 Ep 21/80 | Train: 0.058362 | Val: 0.106575 | LR: 1.69e-05 | TF: 0.10 | NP: 0.52
+[13:18:51] P1 Ep 22/80 | Train: 0.057726 | Val: 0.105803 | LR: 1.67e-05 | TF: 0.10 | NP: 0.52
+[13:20:08] P1 Ep 23/80 | Train: 0.057765 | Val: 0.106002 | LR: 1.64e-05 | TF: 0.09 | NP: 0.51
+[13:21:22] P1 Ep 24/80 | Train: 0.056851 | Val: 0.106002 | LR: 1.61e-05 | TF: 0.09 | NP: 0.50
+[13:22:36] P1 Ep 25/80 | Train: 0.056944 | Val: 0.105939 | LR: 1.58e-05 | TF: 0.08 | NP: 0.49
+[13:23:52] P1 Ep 26/80 | Train: 0.056835 | Val: 0.105778 | LR: 1.55e-05 | TF: 0.08 | NP: 0.48
+[13:25:10] P1 Ep 27/80 | Train: 0.056351 | Val: 0.105987 | LR: 1.51e-05 | TF: 0.07 | NP: 0.47
+[13:26:27] P1 Ep 28/80 | Train: 0.057155 | Val: 0.105820 | LR: 1.48e-05 | TF: 0.06 | NP: 0.46
+[13:27:42] P1 Ep 29/80 | Train: 0.056522 | Val: 0.105906 | LR: 1.45e-05 | TF: 0.06 | NP: 0.45
+[13:28:56] P1 Ep 30/80 | Train: 0.057007 | Val: 0.105835 | LR: 1.41e-05 | TF: 0.06 | NP: 0.45
+[13:30:11] P1 Ep 31/80 | Train: 0.056296 | Val: 0.105715 | LR: 1.38e-05 | TF: 0.05 | NP: 0.44
+[13:31:23] P1 Ep 32/80 | Train: 0.056693 | Val: 0.105578 | LR: 1.34e-05 | TF: 0.04 | NP: 0.43
+[13:32:37] P1 Ep 33/80 | Train: 0.056073 | Val: 0.105810 | LR: 1.31e-05 | TF: 0.04 | NP: 0.42
+[13:33:50] P1 Ep 34/80 | Train: 0.056355 | Val: 0.105725 | LR: 1.27e-05 | TF: 0.04 | NP: 0.41
+[13:35:03] P1 Ep 35/80 | Train: 0.056092 | Val: 0.105561 | LR: 1.24e-05 | TF: 0.03 | NP: 0.40
+[13:36:16] P1 Ep 36/80 | Train: 0.056128 | Val: 0.105912 | LR: 1.20e-05 | TF: 0.03 | NP: 0.39
+[13:37:28] P1 Ep 37/80 | Train: 0.056166 | Val: 0.105963 | LR: 1.16e-05 | TF: 0.02 | NP: 0.39
+[13:38:37] P1 Ep 38/80 | Train: 0.055460 | Val: 0.106817 | LR: 1.12e-05 | TF: 0.01 | NP: 0.38
+[13:39:49] P1 Ep 39/80 | Train: 0.055705 | Val: 0.106077 | LR: 1.09e-05 | TF: 0.01 | NP: 0.37
+[13:40:58] P1 Ep 40/80 | Train: 0.055776 | Val: 0.106179 | LR: 1.05e-05 | TF: 0.01 | NP: 0.36
+[13:42:03] P1 Ep 41/80 | Train: 0.055199 | Val: 0.105851 | LR: 1.01e-05 | TF: 0.00 | NP: 0.35
+[13:43:09] P1 Ep 42/80 | Train: 0.054663 | Val: 0.105944 | LR: 9.75e-06 | TF: 0.00 | NP: 0.34
+[13:44:14] P1 Ep 43/80 | Train: 0.054780 | Val: 0.106381 | LR: 9.38e-06 | TF: 0.00 | NP: 0.33
+[13:45:20] P1 Ep 44/80 | Train: 0.054248 | Val: 0.106289 | LR: 9.01e-06 | TF: 0.00 | NP: 0.32
+[13:46:26] P1 Ep 45/80 | Train: 0.054735 | Val: 0.106070 | LR: 8.65e-06 | TF: 0.00 | NP: 0.31
+[13:47:29] P1 Ep 46/80 | Train: 0.054244 | Val: 0.106063 | LR: 8.28e-06 | TF: 0.00 | NP: 0.31
+[13:48:32] P1 Ep 47/80 | Train: 0.053652 | Val: 0.106781 | LR: 7.92e-06 | TF: 0.00 | NP: 0.30
+[13:49:35] P1 Ep 48/80 | Train: 0.053399 | Val: 0.106223 | LR: 7.56e-06 | TF: 0.00 | NP: 0.29
+[13:50:39] P1 Ep 49/80 | Train: 0.053571 | Val: 0.106390 | LR: 7.21e-06 | TF: 0.00 | NP: 0.28
+[13:51:41] P1 Ep 50/80 | Train: 0.053090 | Val: 0.106306 | LR: 6.86e-06 | TF: 0.00 | NP: 0.27
+[13:52:42] P1 Ep 51/80 | Train: 0.052787 | Val: 0.106494 | LR: 6.52e-06 | TF: 0.00 | NP: 0.26
+[13:53:41] P1 Ep 52/80 | Train: 0.052288 | Val: 0.106509 | LR: 6.19e-06 | TF: 0.00 | NP: 0.25
+[13:54:41] P1 Ep 53/80 | Train: 0.052388 | Val: 0.106455 | LR: 5.86e-06 | TF: 0.00 | NP: 0.24
+[13:55:37] P1 Ep 54/80 | Train: 0.052128 | Val: 0.106198 | LR: 5.54e-06 | TF: 0.00 | NP: 0.24
+[13:56:32] P1 Ep 55/80 | Train: 0.051960 | Val: 0.106368 | LR: 5.22e-06 | TF: 0.00 | NP: 0.23
+[13:57:26] P1 Ep 56/80 | Train: 0.051742 | Val: 0.106085 | LR: 4.92e-06 | TF: 0.00 | NP: 0.22
+[13:58:19] P1 Ep 57/80 | Train: 0.051427 | Val: 0.106450 | LR: 4.62e-06 | TF: 0.00 | NP: 0.21
+[13:59:14] P1 Ep 58/80 | Train: 0.050891 | Val: 0.106534 | LR: 4.33e-06 | TF: 0.00 | NP: 0.20
+[14:00:10] P1 Ep 59/80 | Train: 0.051350 | Val: 0.106493 | LR: 4.05e-06 | TF: 0.00 | NP: 0.20
+[14:01:04] P1 Ep 60/80 | Train: 0.051454 | Val: 0.106614 | LR: 3.78e-06 | TF: 0.00 | NP: 0.20
+[14:01:59] P1 Ep 61/80 | Train: 0.050717 | Val: 0.106441 | LR: 3.52e-06 | TF: 0.00 | NP: 0.20
+[14:02:55] P1 Ep 62/80 | Train: 0.051467 | Val: 0.106609 | LR: 3.28e-06 | TF: 0.00 | NP: 0.20
+[14:03:50] P1 Ep 63/80 | Train: 0.050955 | Val: 0.106522 | LR: 3.04e-06 | TF: 0.00 | NP: 0.20
+[14:04:44] P1 Ep 64/80 | Train: 0.051030 | Val: 0.106480 | LR: 2.81e-06 | TF: 0.00 | NP: 0.20
+[14:05:40] P1 Ep 65/80 | Train: 0.051180 | Val: 0.106825 | LR: 2.60e-06 | TF: 0.00 | NP: 0.20
+[14:06:34] P1 Ep 66/80 | Train: 0.051234 | Val: 0.106796 | LR: 2.40e-06 | TF: 0.00 | NP: 0.20
+[14:07:29] P1 Ep 67/80 | Train: 0.051035 | Val: 0.106585 | LR: 2.21e-06 | TF: 0.00 | NP: 0.20
+[14:08:25] P1 Ep 68/80 | Train: 0.050935 | Val: 0.106445 | LR: 2.04e-06 | TF: 0.00 | NP: 0.20
+[14:09:18] P1 Ep 69/80 | Train: 0.050815 | Val: 0.106700 | LR: 1.87e-06 | TF: 0.00 | NP: 0.20
+[14:10:10] P1 Ep 70/80 | Train: 0.050347 | Val: 0.106777 | LR: 1.72e-06 | TF: 0.00 | NP: 0.20
+[14:11:06] P1 Ep 71/80 | Train: 0.051426 | Val: 0.106826 | LR: 1.59e-06 | TF: 0.00 | NP: 0.20
+[14:11:59] P1 Ep 72/80 | Train: 0.050445 | Val: 0.106671 | LR: 1.46e-06 | TF: 0.00 | NP: 0.20
+[14:12:52] P1 Ep 73/80 | Train: 0.051250 | Val: 0.106840 | LR: 1.36e-06 | TF: 0.00 | NP: 0.20
+[14:13:42] P1 Ep 74/80 | Train: 0.050837 | Val: 0.106714 | LR: 1.26e-06 | TF: 0.00 | NP: 0.20
+[14:14:33] P1 Ep 75/80 | Train: 0.050271 | Val: 0.106559 | LR: 1.18e-06 | TF: 0.00 | NP: 0.20
+[14:15:28] P1 Ep 76/80 | Train: 0.050798 | Val: 0.106617 | LR: 1.12e-06 | TF: 0.00 | NP: 0.20
+[14:16:23] P1 Ep 77/80 | Train: 0.051059 | Val: 0.106779 | LR: 1.07e-06 | TF: 0.00 | NP: 0.20
+[14:17:17] P1 Ep 78/80 | Train: 0.050716 | Val: 0.106623 | LR: 1.03e-06 | TF: 0.00 | NP: 0.20
+[14:18:12] P1 Ep 79/80 | Train: 0.050641 | Val: 0.106691 | LR: 1.01e-06 | TF: 0.00 | NP: 0.20
+[14:19:06] P1 Ep 80/80 | Train: 0.050322 | Val: 0.106673 | LR: 1.00e-06 | TF: 0.00 | NP: 0.20
+[14:19:07] === Phase 2: 8-step AR with noise decay (150 epochs) ===
+[14:19:13] Train: 5549, Val: 682
+[14:21:09] P2 Ep 1/150 | Train: 0.101754 | Val: 0.152433 | LR: 1.00e-05 | NP: 0.50
+[14:21:09]   -> Saved P2 (val=0.152433)
+[14:23:00] P2 Ep 2/150 | Train: 0.099688 | Val: 0.152274 | LR: 1.00e-05 | NP: 0.49
+[14:23:00]   -> Saved P2 (val=0.152274)
+[14:24:49] P2 Ep 3/150 | Train: 0.099752 | Val: 0.152190 | LR: 9.99e-06 | NP: 0.49
+[14:24:49]   -> Saved P2 (val=0.152190)
+[14:26:38] P2 Ep 4/150 | Train: 0.098640 | Val: 0.151499 | LR: 9.98e-06 | NP: 0.48
+[14:26:38]   -> Saved P2 (val=0.151499)
+[14:28:28] P2 Ep 5/150 | Train: 0.097336 | Val: 0.152197 | LR: 9.98e-06 | NP: 0.47
+[14:30:22] P2 Ep 6/150 | Train: 0.097932 | Val: 0.151221 | LR: 9.96e-06 | NP: 0.47
+[14:30:22]   -> Saved P2 (val=0.151221)
+[14:32:16] P2 Ep 7/150 | Train: 0.097483 | Val: 0.151522 | LR: 9.95e-06 | NP: 0.46
+[14:34:10] P2 Ep 8/150 | Train: 0.096711 | Val: 0.151732 | LR: 9.94e-06 | NP: 0.45
+[14:35:56] P2 Ep 9/150 | Train: 0.096096 | Val: 0.152045 | LR: 9.92e-06 | NP: 0.45
+[14:37:41] P2 Ep 10/150 | Train: 0.095196 | Val: 0.152588 | LR: 9.90e-06 | NP: 0.44
+[14:39:25] P2 Ep 11/150 | Train: 0.095640 | Val: 0.151398 | LR: 9.88e-06 | NP: 0.43
+[14:41:10] P2 Ep 12/150 | Train: 0.094483 | Val: 0.151931 | LR: 9.86e-06 | NP: 0.43
+[14:46:21] P2 Ep 13/150 | Train: 0.093845 | Val: 0.151876 | LR: 9.83e-06 | NP: 0.42
+[14:51:11] P2 Ep 14/150 | Train: 0.093061 | Val: 0.152407 | LR: 9.81e-06 | NP: 0.41
+[14:52:52] P2 Ep 15/150 | Train: 0.093344 | Val: 0.152210 | LR: 9.78e-06 | NP: 0.41
+[14:54:32] P2 Ep 16/150 | Train: 0.092273 | Val: 0.152266 | LR: 9.75e-06 | NP: 0.40
+[14:56:21] P2 Ep 17/150 | Train: 0.092344 | Val: 0.152521 | LR: 9.72e-06 | NP: 0.39
+[14:58:07] P2 Ep 18/150 | Train: 0.092146 | Val: 0.153587 | LR: 9.68e-06 | NP: 0.39
+[14:59:53] P2 Ep 19/150 | Train: 0.091652 | Val: 0.152774 | LR: 9.65e-06 | NP: 0.38
+[15:01:36] P2 Ep 20/150 | Train: 0.090168 | Val: 0.152504 | LR: 9.61e-06 | NP: 0.37
+[15:03:15] P2 Ep 21/150 | Train: 0.091359 | Val: 0.152312 | LR: 9.57e-06 | NP: 0.37
+[15:04:54] P2 Ep 22/150 | Train: 0.090870 | Val: 0.153649 | LR: 9.53e-06 | NP: 0.36
+[15:06:32] P2 Ep 23/150 | Train: 0.089860 | Val: 0.152573 | LR: 9.49e-06 | NP: 0.35
+[15:08:12] P2 Ep 24/150 | Train: 0.089024 | Val: 0.153130 | LR: 9.44e-06 | NP: 0.35
+[15:09:55] P2 Ep 25/150 | Train: 0.088663 | Val: 0.152396 | LR: 9.40e-06 | NP: 0.34
+[15:11:35] P2 Ep 26/150 | Train: 0.088985 | Val: 0.153101 | LR: 9.35e-06 | NP: 0.33
+[15:13:17] P2 Ep 27/150 | Train: 0.088276 | Val: 0.153554 | LR: 9.30e-06 | NP: 0.33
+[15:14:55] P2 Ep 28/150 | Train: 0.087718 | Val: 0.152368 | LR: 9.25e-06 | NP: 0.32
+[15:16:29] P2 Ep 29/150 | Train: 0.086924 | Val: 0.152428 | LR: 9.20e-06 | NP: 0.31
+[15:17:59] P2 Ep 30/150 | Train: 0.086764 | Val: 0.153271 | LR: 9.14e-06 | NP: 0.31
+[15:19:31] P2 Ep 31/150 | Train: 0.087409 | Val: 0.153467 | LR: 9.08e-06 | NP: 0.30
+[15:21:02] P2 Ep 32/150 | Train: 0.086298 | Val: 0.152615 | LR: 9.03e-06 | NP: 0.29
+[15:24:37] P2 Ep 33/150 | Train: 0.085762 | Val: 0.153204 | LR: 8.97e-06 | NP: 0.29
+[15:27:11] P2 Ep 34/150 | Train: 0.085166 | Val: 0.152645 | LR: 8.91e-06 | NP: 0.28
+[15:28:46] P2 Ep 35/150 | Train: 0.085598 | Val: 0.153063 | LR: 8.84e-06 | NP: 0.27
+[15:30:15] P2 Ep 36/150 | Train: 0.084653 | Val: 0.152888 | LR: 8.78e-06 | NP: 0.27
+[15:31:42] P2 Ep 37/150 | Train: 0.084774 | Val: 0.153305 | LR: 8.72e-06 | NP: 0.26
+[15:33:09] P2 Ep 38/150 | Train: 0.084202 | Val: 0.153384 | LR: 8.65e-06 | NP: 0.25
+[15:34:35] P2 Ep 39/150 | Train: 0.083539 | Val: 0.152966 | LR: 8.58e-06 | NP: 0.25
+[15:36:05] P2 Ep 40/150 | Train: 0.083781 | Val: 0.152655 | LR: 8.51e-06 | NP: 0.24
+[15:37:38] P2 Ep 41/150 | Train: 0.083613 | Val: 0.152845 | LR: 8.44e-06 | NP: 0.23
+[15:39:07] P2 Ep 42/150 | Train: 0.082472 | Val: 0.152577 | LR: 8.37e-06 | NP: 0.23
+[15:40:36] P2 Ep 43/150 | Train: 0.082296 | Val: 0.154262 | LR: 8.30e-06 | NP: 0.22
+[15:42:02] P2 Ep 44/150 | Train: 0.082341 | Val: 0.154107 | LR: 8.22e-06 | NP: 0.21
+[15:43:29] P2 Ep 45/150 | Train: 0.082123 | Val: 0.153391 | LR: 8.15e-06 | NP: 0.21
+[15:44:52] P2 Ep 46/150 | Train: 0.081416 | Val: 0.153467 | LR: 8.07e-06 | NP: 0.20
+[15:46:13] P2 Ep 47/150 | Train: 0.080997 | Val: 0.154717 | LR: 7.99e-06 | NP: 0.19
+[15:47:34] P2 Ep 48/150 | Train: 0.080812 | Val: 0.153156 | LR: 7.91e-06 | NP: 0.19
+[15:48:53] P2 Ep 49/150 | Train: 0.080396 | Val: 0.153658 | LR: 7.83e-06 | NP: 0.18
+[15:50:13] P2 Ep 50/150 | Train: 0.080431 | Val: 0.153539 | LR: 7.75e-06 | NP: 0.17
+[15:51:35] P2 Ep 51/150 | Train: 0.079722 | Val: 0.153832 | LR: 7.67e-06 | NP: 0.17
+[15:52:56] P2 Ep 52/150 | Train: 0.078960 | Val: 0.153832 | LR: 7.58e-06 | NP: 0.16
+[15:54:17] P2 Ep 53/150 | Train: 0.079002 | Val: 0.154286 | LR: 7.50e-06 | NP: 0.15
+[15:55:36] P2 Ep 54/150 | Train: 0.078540 | Val: 0.153657 | LR: 7.42e-06 | NP: 0.15
+[15:56:56] P2 Ep 55/150 | Train: 0.078308 | Val: 0.153990 | LR: 7.33e-06 | NP: 0.14
+[16:00:48] P2 Ep 56/150 | Train: 0.078187 | Val: 0.154068 | LR: 7.24e-06 | NP: 0.13
+[16:02:02] P2 Ep 57/150 | Train: 0.077911 | Val: 0.154126 | LR: 7.16e-06 | NP: 0.13
+[16:03:16] P2 Ep 58/150 | Train: 0.077579 | Val: 0.154452 | LR: 7.07e-06 | NP: 0.12
+[16:04:30] P2 Ep 59/150 | Train: 0.077332 | Val: 0.153857 | LR: 6.98e-06 | NP: 0.11
+[16:05:43] P2 Ep 60/150 | Train: 0.076650 | Val: 0.153625 | LR: 6.89e-06 | NP: 0.11
+[16:06:57] P2 Ep 61/150 | Train: 0.076305 | Val: 0.154083 | LR: 6.80e-06 | NP: 0.10
+[16:08:09] P2 Ep 62/150 | Train: 0.075645 | Val: 0.154870 | LR: 6.71e-06 | NP: 0.09
+[16:09:21] P2 Ep 63/150 | Train: 0.075782 | Val: 0.154309 | LR: 6.62e-06 | NP: 0.09
+[16:10:31] P2 Ep 64/150 | Train: 0.075117 | Val: 0.155150 | LR: 6.53e-06 | NP: 0.08
+[16:11:42] P2 Ep 65/150 | Train: 0.074917 | Val: 0.154863 | LR: 6.44e-06 | NP: 0.07
+[16:12:51] P2 Ep 66/150 | Train: 0.074718 | Val: 0.155326 | LR: 6.34e-06 | NP: 0.07
+[16:14:00] P2 Ep 67/150 | Train: 0.074404 | Val: 0.154637 | LR: 6.25e-06 | NP: 0.06
+[16:15:08] P2 Ep 68/150 | Train: 0.073795 | Val: 0.154473 | LR: 6.16e-06 | NP: 0.05
+[16:16:15] P2 Ep 69/150 | Train: 0.073667 | Val: 0.154415 | LR: 6.06e-06 | NP: 0.05
+[16:17:23] P2 Ep 70/150 | Train: 0.073131 | Val: 0.154870 | LR: 5.97e-06 | NP: 0.04
+[16:18:30] P2 Ep 71/150 | Train: 0.072894 | Val: 0.154363 | LR: 5.88e-06 | NP: 0.03
+[16:19:36] P2 Ep 72/150 | Train: 0.072479 | Val: 0.154667 | LR: 5.78e-06 | NP: 0.03
+[16:20:42] P2 Ep 73/150 | Train: 0.072346 | Val: 0.154891 | LR: 5.69e-06 | NP: 0.02
+[16:21:48] P2 Ep 74/150 | Train: 0.071960 | Val: 0.154338 | LR: 5.59e-06 | NP: 0.01
+[16:22:53] P2 Ep 75/150 | Train: 0.071490 | Val: 0.154866 | LR: 5.50e-06 | NP: 0.01
+[16:23:58] P2 Ep 76/150 | Train: 0.071118 | Val: 0.154872 | LR: 5.41e-06
+[16:25:03] P2 Ep 77/150 | Train: 0.071332 | Val: 0.155365 | LR: 5.31e-06
+[16:26:08] P2 Ep 78/150 | Train: 0.071061 | Val: 0.154417 | LR: 5.22e-06
+[16:27:12] P2 Ep 79/150 | Train: 0.070547 | Val: 0.155742 | LR: 5.12e-06
+[16:28:18] P2 Ep 80/150 | Train: 0.070975 | Val: 0.155047 | LR: 5.03e-06
+[16:29:23] P2 Ep 81/150 | Train: 0.070541 | Val: 0.155869 | LR: 4.94e-06
+[16:30:29] P2 Ep 82/150 | Train: 0.070462 | Val: 0.154952 | LR: 4.84e-06
+[16:31:33] P2 Ep 83/150 | Train: 0.070009 | Val: 0.155245 | LR: 4.75e-06
+[16:32:38] P2 Ep 84/150 | Train: 0.070098 | Val: 0.155800 | LR: 4.66e-06
+[16:33:46] P2 Ep 85/150 | Train: 0.070044 | Val: 0.155692 | LR: 4.56e-06
+[16:34:54] P2 Ep 86/150 | Train: 0.069939 | Val: 0.155624 | LR: 4.47e-06
+[16:36:03] P2 Ep 87/150 | Train: 0.069929 | Val: 0.155902 | LR: 4.38e-06
+[16:37:10] P2 Ep 88/150 | Train: 0.069614 | Val: 0.155839 | LR: 4.29e-06
+[16:38:15] P2 Ep 89/150 | Train: 0.069634 | Val: 0.156251 | LR: 4.20e-06
+[16:39:20] P2 Ep 90/150 | Train: 0.069615 | Val: 0.156350 | LR: 4.11e-06
+[16:40:26] P2 Ep 91/150 | Train: 0.069264 | Val: 0.156132 | LR: 4.02e-06
+[16:41:32] P2 Ep 92/150 | Train: 0.069372 | Val: 0.156132 | LR: 3.93e-06
+[16:42:37] P2 Ep 93/150 | Train: 0.069419 | Val: 0.156336 | LR: 3.84e-06
+[16:43:42] P2 Ep 94/150 | Train: 0.069121 | Val: 0.156934 | LR: 3.76e-06
+[16:44:47] P2 Ep 95/150 | Train: 0.068904 | Val: 0.156873 | LR: 3.67e-06
+[16:45:52] P2 Ep 96/150 | Train: 0.068944 | Val: 0.156756 | LR: 3.58e-06
+[16:46:58] P2 Ep 97/150 | Train: 0.068881 | Val: 0.156877 | LR: 3.50e-06
+[16:48:01] P2 Ep 98/150 | Train: 0.068693 | Val: 0.157147 | LR: 3.42e-06
+[16:49:07] P2 Ep 99/150 | Train: 0.068702 | Val: 0.157339 | LR: 3.33e-06
+[16:50:14] P2 Ep 100/150 | Train: 0.068535 | Val: 0.157586 | LR: 3.25e-06
+[16:51:19] P2 Ep 101/150 | Train: 0.068474 | Val: 0.157016 | LR: 3.17e-06
+[16:52:23] P2 Ep 102/150 | Train: 0.068581 | Val: 0.157132 | LR: 3.09e-06
+[16:53:29] P2 Ep 103/150 | Train: 0.068578 | Val: 0.156982 | LR: 3.01e-06
+[16:54:35] P2 Ep 104/150 | Train: 0.068317 | Val: 0.157427 | LR: 2.93e-06
+[16:55:42] P2 Ep 105/150 | Train: 0.068424 | Val: 0.157384 | LR: 2.85e-06
+[16:56:48] P2 Ep 106/150 | Train: 0.068248 | Val: 0.157696 | LR: 2.78e-06
+[16:57:54] P2 Ep 107/150 | Train: 0.068074 | Val: 0.157475 | LR: 2.70e-06
+[16:59:00] P2 Ep 108/150 | Train: 0.068216 | Val: 0.157248 | LR: 2.63e-06
+[17:00:07] P2 Ep 109/150 | Train: 0.068056 | Val: 0.157959 | LR: 2.56e-06
+[17:01:12] P2 Ep 110/150 | Train: 0.067960 | Val: 0.157656 | LR: 2.49e-06
+[17:02:19] P2 Ep 111/150 | Train: 0.068169 | Val: 0.157431 | LR: 2.42e-06
+[17:03:24] P2 Ep 112/150 | Train: 0.067761 | Val: 0.158117 | LR: 2.35e-06
+[17:04:31] P2 Ep 113/150 | Train: 0.067777 | Val: 0.158170 | LR: 2.28e-06
+[17:05:37] P2 Ep 114/150 | Train: 0.067623 | Val: 0.157751 | LR: 2.22e-06
+[17:06:43] P2 Ep 115/150 | Train: 0.067741 | Val: 0.157428 | LR: 2.16e-06
+[17:07:50] P2 Ep 116/150 | Train: 0.067588 | Val: 0.157786 | LR: 2.09e-06
+[17:08:57] P2 Ep 117/150 | Train: 0.067652 | Val: 0.158238 | LR: 2.03e-06
+[17:10:04] P2 Ep 118/150 | Train: 0.067387 | Val: 0.158239 | LR: 1.97e-06
+[17:11:11] P2 Ep 119/150 | Train: 0.067485 | Val: 0.158385 | LR: 1.92e-06
+[17:12:17] P2 Ep 120/150 | Train: 0.067419 | Val: 0.158199 | LR: 1.86e-06
+[17:13:24] P2 Ep 121/150 | Train: 0.067331 | Val: 0.158202 | LR: 1.80e-06
+[17:14:31] P2 Ep 122/150 | Train: 0.067393 | Val: 0.158109 | LR: 1.75e-06
+[17:15:37] P2 Ep 123/150 | Train: 0.067470 | Val: 0.158638 | LR: 1.70e-06
+[17:16:43] P2 Ep 124/150 | Train: 0.067511 | Val: 0.158226 | LR: 1.65e-06
+[17:17:49] P2 Ep 125/150 | Train: 0.067258 | Val: 0.158870 | LR: 1.60e-06
+[17:18:55] P2 Ep 126/150 | Train: 0.067368 | Val: 0.158049 | LR: 1.56e-06
+[17:20:02] P2 Ep 127/150 | Train: 0.067312 | Val: 0.158657 | LR: 1.51e-06
+[17:21:08] P2 Ep 128/150 | Train: 0.067118 | Val: 0.159119 | LR: 1.47e-06
+[17:22:15] P2 Ep 129/150 | Train: 0.067190 | Val: 0.158437 | LR: 1.43e-06
+[17:23:22] P2 Ep 130/150 | Train: 0.066941 | Val: 0.158701 | LR: 1.39e-06
+[17:24:28] P2 Ep 131/150 | Train: 0.067152 | Val: 0.158655 | LR: 1.35e-06
+[17:25:35] P2 Ep 132/150 | Train: 0.067221 | Val: 0.158538 | LR: 1.32e-06
+[17:26:42] P2 Ep 133/150 | Train: 0.067087 | Val: 0.158500 | LR: 1.28e-06
+[17:27:48] P2 Ep 134/150 | Train: 0.067050 | Val: 0.159203 | LR: 1.25e-06
+[17:28:55] P2 Ep 135/150 | Train: 0.067026 | Val: 0.159071 | LR: 1.22e-06
+[17:30:01] P2 Ep 136/150 | Train: 0.066866 | Val: 0.159042 | LR: 1.19e-06
+[17:31:08] P2 Ep 137/150 | Train: 0.066927 | Val: 0.159214 | LR: 1.17e-06
+[17:32:14] P2 Ep 138/150 | Train: 0.066824 | Val: 0.158977 | LR: 1.14e-06
+[17:33:20] P2 Ep 139/150 | Train: 0.066809 | Val: 0.159073 | LR: 1.12e-06
+[17:34:27] P2 Ep 140/150 | Train: 0.066978 | Val: 0.159100 | LR: 1.10e-06
+[17:35:33] P2 Ep 141/150 | Train: 0.066976 | Val: 0.159303 | LR: 1.08e-06
+[17:36:39] P2 Ep 142/150 | Train: 0.066872 | Val: 0.158624 | LR: 1.06e-06
+[17:37:45] P2 Ep 143/150 | Train: 0.066707 | Val: 0.159305 | LR: 1.05e-06
+[17:38:52] P2 Ep 144/150 | Train: 0.066753 | Val: 0.159082 | LR: 1.04e-06
+[17:39:58] P2 Ep 145/150 | Train: 0.066867 | Val: 0.159045 | LR: 1.02e-06
+[17:41:05] P2 Ep 146/150 | Train: 0.066663 | Val: 0.158994 | LR: 1.02e-06
+[17:42:11] P2 Ep 147/150 | Train: 0.066745 | Val: 0.159186 | LR: 1.01e-06
+[17:43:17] P2 Ep 148/150 | Train: 0.066781 | Val: 0.159554 | LR: 1.00e-06
+[17:44:21] P2 Ep 149/150 | Train: 0.066731 | Val: 0.159409 | LR: 1.00e-06
+[17:45:24] P2 Ep 150/150 | Train: 0.066684 | Val: 0.158994 | LR: 1.00e-06
+[17:45:24] Cleaned up error patterns.
+[17:45:24] Training complete.