Upload Main_200003

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_005000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf780ffedf8689f164c00b01604d3fa320388a40244b6b6059f13af95aa6bdb7
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_010000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c1a09acbcf0fe687cb88ea0d03384132f11491a7096226ffffce2ddb5a65c02
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_015000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeee0418fb43874438b2ec7ce7987ab7c2fc0c821660a8521f95589f4b6d60ea
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_020000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:171021c6931f55a59a50d59a02408a56bb43cad2de64dcb43b76efd123bb824d
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_025000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e691f8f2cf4ff2a8baace32eb3faef0c366bf180fb01857edcd9805f736b02
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_030000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4ce08683280fe98fbb478e39076857eecbca88581825bfb5b88c9b86a3463db
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_035000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0be1b1e3d9f137b6ba131317c98e6eac5bdc4588e0115b2d6482691660369884
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_040000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2aac929c7893a4c068cdd2b9131f04c9faf3554454c6a5b7edf5f84b3e6bda0
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_045000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b90b959f2b911283e3c075f9837ef51059db4a4bcd23dbfd80e7694dcb3bb9e
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_050000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff370a0eed00a8b466ffdeea2ca6890eab0cd7b3b0c8df0f401eaead634a6930
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_055000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8c88f0927765376a3476b354cd3a603d3b79b5aa547a890b26bba0ec59a8351
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_060000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c27efcb8515056904d818705f2876a835584316166feeaa703ae9b729ed015b3
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_065000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d13c6fbfd9afadffae54fda69c5bc2b6b5cf172910b4e329765ef1d0b95feb6e
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_070000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2820bfd429f3ba6b66033debf9f9dbf0469ab50449cde6027930e62d9eb8230
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_075000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56fa6d050540aa44f4613247f90f3cc9ae137f84137d4c0d7a3b4deeb7b8a664
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_080000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d202a0b77faab6eb2dbe7043e511b1c7f3331edc1ecd50265556fbd34d8bc25f
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_085000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60c39f90a6d0ce5a0888860d6788c16c7f3dc575b8537d2226430a0a5c71bbf3
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_090000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05a57fba88ff9b76926afb7dfa2bc608ca7baca29c7576794a817e1eded5140
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_095000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2396b4925f09c9e9cbfd50a3f0666b954bae7f0bd2068cedb19685ebf5355625
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_100000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36b38c1885f1c9a0f0a9fc26ee1951e8f5ae481f2e48424baf442d9a2b2eebb2
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_105000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:862fe1414fdce7e8033417aec971c496f6274f806d62f5b1fc5436dd832386a4
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_110000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55b2554934ef3616e79e5993b18b64ecfab90bc84a3f3428f7decbe26df4e1cc
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_115000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d57ffec3b32c9f77c718e799b63de0336de8b39891477f1389dd359386b0124b
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_120000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de4608681855baeb0dd09b303bbd3241adad8136aecb66ef1be4d3d30282b95f
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_125000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:812f53d3460c5e7e55a1fd616ded1d07c8784aebf630729a2d9a7e4898653d12
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_130000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0332478c95dac46cc1428f7e8bab84650c6316c973963795ac55635c8651ac57
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_135000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cae9d91af2c1400cb78601d013e0da16210f966ecc661cf0db9dcfd09dc7b68e
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_140000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0226306a6fd7422c9484e986b56bb2d0a21c566a0b36d78c1978bd4759c90bd8
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_145000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5a7e9a071cbb91ad4f3406c8a50dc667754b5851259c9b1e23aafe3fa7723a2
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_150000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69a63b2fcefd5eb20ad8d6289cfca9d6026cf2e1f2971cf119de4936a8a786fe
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_155000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68c4c189d6dee90fcc54460b16fbee964dd275a19511dae5157f359384a774b
+size 151982040

Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4f83839a2ef74f2448d4b9391a28e5703bcb6f69e631d5535dc19475678a07d
+size 151985051

Main_200003/937fc7f323a4d7d3ae9480105345ef67/cfg.txt

@@ -0,0 +1,55 @@
+DESCRIPTION: TimeMoE Base
+DEVICE: gpu
+DEVICE_NUM: 4
+RUNNER: <class 'baselines.TimeMoE4.runner.runner.TimeMoERunner'>
+MODEL:
+  NAME: TimeMoE4
+  ARCH: <class 'baselines.TimeMoE4.arch.timemoe.TimeMoE4'>
+  PARAM:
+    model_id: baselines/TimeMoE/ckpt/TimeMoE-50M
+    from_pretrained: False
+    context_length: 4079
+    trust_remote_code: True
+  DTYPE: bfloat16
+METRICS:
+  FUNCS:
+TRAIN:
+  COMPILE_MODEL: True
+  NUM_ITERATIONS: 200003
+  CKPT_SAVE_DIR: checkpoints/TimeMoE4/Main_200003
+  CKPT_SAVE_STRATEGY: 5000
+  LOSS: fake_loss
+  OPTIM:
+    TYPE: AdamW
+    PARAM:
+      lr: 0.0004
+      betas: (0.9, 0.95)
+      fused: True
+  LR_SCHEDULER:
+    TYPE: CosineWarmup
+    PARAM:
+      num_warmup_steps: 10000
+      num_training_steps: 200003
+  CLIP_GRAD_PARAM:
+    max_norm: 1.0
+  DATA:
+    BATCH_SIZE: 64
+    SHUFFLE: True
+    PIN_MEMORY: True
+    PREFETCH: True
+  GRAD_ACCUMULATION_STEPS: 1
+VAL:
+  INTERVAL: 5000
+  DATA:
+    BATCH_SIZE: 128
+EVAL:
+  USE_GPU: True
+DATASET:
+  NAME: Main
+  TYPE: <class 'baselines.TimeMoE4.data.mix_dataset_v2.MixedSourceDataset_v2'>
+  PARAM:
+    num_valid_samples: 1000
+INFERENCE:
+  GENERATION_PARAMS:
+    normalize: True
+MD5: 937fc7f323a4d7d3ae9480105345ef67

Main_200003/937fc7f323a4d7d3ae9480105345ef67/tensorboard/events.out.tfevents.1767768860.brev-5x9knwe1p.1631682.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4112a746ba65eb6c329667113d7356a9113cb0e290d616aed45c89d9890924f6
+size 32418418

Main_200003/937fc7f323a4d7d3ae9480105345ef67/timemoe_base.py

@@ -0,0 +1,125 @@
+# 采样概率变化
+
+import os
+import sys
+from easydict import EasyDict
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ..arch import TimeMoE4
+from ..data import MixedSourceDataset_v2
+from ..runner import TimeMoERunner
+from ..loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+pretrained = False  # Whether to use a pretrained model
+
+MODEL_ARCH = TimeMoE4
+
+MODEL_PARAM = {
+    'model_id': "baselines/TimeMoE/ckpt/TimeMoE-50M",
+    'from_pretrained': pretrained,
+    'context_length': 4079,
+    'trust_remote_code': True,
+}
+DATA_NAME = "Main"
+
+# N = 20_000_000
+# batch size = 16*8
+# 20_000_000 / 16 / 8 = 156250 iterations
+# 20_000_000 * 4096 / 16 / 8 / 4096 = 156_250
+
+NUM_ITERATIONS = 200_003 # 总轮数   20_000_000 * 4096 / 16 / 4 / 4096 = 312,500
+VAL_ITERATION_INTERVAL = 5_000 # 每VAL_ITERATION_INTERVAL执行一次验证
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'TimeMoE Base'
+CFG.DEVICE = 'gpu'
+CFG.DEVICE_NUM = 4
+# Runner
+CFG.RUNNER = TimeMoERunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+# CFG.MODEL.DTYPE= 'float32'
+
+############################## Metrics Configuration ##############################
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({})
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = True
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+CFG.TRAIN.CKPT_SAVE_STRATEGY = VAL_ITERATION_INTERVAL * 1 # 保存策略，每VAL_ITERATION_INTERVAL * 5保存一次模型
+CFG.TRAIN.LOSS = fake_loss
+# Optimizer settings
+CFG.TRAIN.OPTIM = EasyDict()
+CFG.TRAIN.OPTIM.TYPE = "AdamW"
+CFG.TRAIN.OPTIM.PARAM = {
+    "lr": 4e-4,
+    "betas": (0.9, 0.95),
+    "fused": True,
+    # "weight_decay": 1e-1,
+}
+# Learning rate scheduler settings
+CFG.TRAIN.LR_SCHEDULER = EasyDict()
+CFG.TRAIN.LR_SCHEDULER.TYPE = "CosineWarmup"
+CFG.TRAIN.LR_SCHEDULER.PARAM = {
+    'num_warmup_steps': int(NUM_ITERATIONS / 100 * 5), # 5%的warmup启动比例
+    'num_training_steps': NUM_ITERATIONS,
+}
+CFG.TRAIN.CLIP_GRAD_PARAM = {
+    'max_norm': 1.0
+}
+# Train data loader settings
+CFG.TRAIN.DATA = EasyDict()
+CFG.TRAIN.DATA.BATCH_SIZE = 64  # 16  /  4
+CFG.TRAIN.DATA.SHUFFLE = True # has to be False
+CFG.TRAIN.DATA.PIN_MEMORY = True
+CFG.TRAIN.DATA.PREFETCH = True
+CFG.TRAIN.GRAD_ACCUMULATION_STEPS = 1
+# CFG.TRAIN.DATA.NUM_WORKERS = 4
+
+############################## Validation Configuration ##############################
+CFG.VAL = EasyDict()
+CFG.VAL.INTERVAL = VAL_ITERATION_INTERVAL
+CFG.VAL.DATA = EasyDict()
+CFG.VAL.DATA.BATCH_SIZE = 128  # 32  /  8
+
+############################## Evaluation Configuration ##############################
+
+CFG.EVAL = EasyDict()
+# Evaluation parameters
+CFG.EVAL.USE_GPU = True # Whether to use GPU for evaluation. Default: True
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = MixedSourceDataset_v2
+CFG.DATASET.PARAM = EasyDict({
+    'num_valid_samples': 1000
+})
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+    'normalize': not pretrained
+})
+

Main_200003/937fc7f323a4d7d3ae9480105345ef67/training_log_20260107065409.log

@@ -0,0 +1,248 @@
+2026-01-07 06:54:09,447 - easytorch-training - INFO - Initializing training.
+2026-01-07 06:54:09,447 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-07 06:54:09,448 - easytorch-training - INFO - Building training data loader.
+2026-01-07 06:54:20,632 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-07 06:54:20,632 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-07 06:54:20,633 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-07 06:54:20,633 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-07 06:54:20,635 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.0004
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-07 06:54:20,635 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x7e51283ca150>
+2026-01-07 06:54:20,637 - easytorch-training - INFO - Initializing validation.
+2026-01-07 06:54:20,638 - easytorch-training - INFO - Building val data loader.
+2026-01-07 06:54:24,141 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-07 06:54:49,987 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-07 06:54:49,987 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-07 06:54:49,987 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-07 06:54:49,988 - easytorch-training - INFO - Number of parameters: 12653568
+2026-01-07 06:54:49,988 - easytorch-training - INFO - Training with 4 GPUs, batch size per GPUs: 64, grad_accumulation_steps: 1
+2026-01-07 06:54:49,989 - easytorch-training - INFO - Effective batch size: 256
+2026-01-07 07:43:41,002 - easytorch-training - INFO - Iteration 5000 / 200003
+2026-01-07 07:43:41,369 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 1.00e-04, train/loss: 3.2162, train/grad_norm: 30.2086, train/amp_scale: 1.0000]
+2026-01-07 07:43:41,370 - easytorch-training - INFO - Start validation.
+2026-01-07 07:43:55,259 - easytorch-training - INFO - Result <val>: [val/time: 13.77 (s), val/loss: 3.3376]
+2026-01-07 07:43:55,371 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 07:43:55,477 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_005000.pt saved
+2026-01-07 07:43:55,478 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:38:31
+2026-01-07 08:47:40,179 - easytorch-training - INFO - Iteration 10000 / 200003
+2026-01-07 08:47:40,551 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.76 (s), train/lr: 3.00e-04, train/loss: 2.1032, train/grad_norm: 4.2165, train/amp_scale: 1.0000]
+2026-01-07 08:47:40,551 - easytorch-training - INFO - Start validation.
+2026-01-07 08:47:47,100 - easytorch-training - INFO - Result <val>: [val/time: 6.43 (s), val/loss: 3.2769]
+2026-01-07 08:47:47,222 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 08:47:47,331 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_010000.pt saved
+2026-01-07 08:47:47,332 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 20:33:58
+2026-01-07 09:37:19,647 - easytorch-training - INFO - Iteration 15000 / 200003
+2026-01-07 09:37:20,017 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 4.00e-04, train/loss: 1.8988, train/grad_norm: 3.5223, train/amp_scale: 1.0000]
+2026-01-07 09:37:20,018 - easytorch-training - INFO - Start validation.
+2026-01-07 09:37:26,549 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 3.2378]
+2026-01-07 09:37:26,666 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 09:37:26,774 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_015000.pt saved
+2026-01-07 09:37:26,774 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 19:03:02
+2026-01-07 10:26:10,634 - easytorch-training - INFO - Iteration 20000 / 200003
+2026-01-07 10:26:11,007 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 3.98e-04, train/loss: 1.8353, train/grad_norm: 3.4002, train/amp_scale: 1.0000]
+2026-01-07 10:26:11,007 - easytorch-training - INFO - Start validation.
+2026-01-07 10:26:17,533 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.2155]
+2026-01-07 10:26:17,650 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 10:26:17,760 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_020000.pt saved
+2026-01-07 10:26:17,761 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 18:09:29
+2026-01-07 11:15:21,133 - easytorch-training - INFO - Iteration 25000 / 200003
+2026-01-07 11:15:21,505 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 3.96e-04, train/loss: 1.7968, train/grad_norm: 3.2657, train/amp_scale: 1.0000]
+2026-01-07 11:15:21,505 - easytorch-training - INFO - Start validation.
+2026-01-07 11:15:28,016 - easytorch-training - INFO - Result <val>: [val/time: 6.39 (s), val/loss: 3.2073]
+2026-01-07 11:15:28,132 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 11:15:28,244 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_025000.pt saved
+2026-01-07 11:15:28,245 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 17:39:57
+2026-01-07 12:04:34,928 - easytorch-training - INFO - Iteration 30000 / 200003
+2026-01-07 12:04:35,300 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 3.92e-04, train/loss: 1.7817, train/grad_norm: 3.2711, train/amp_scale: 1.0000]
+2026-01-07 12:04:35,301 - easytorch-training - INFO - Start validation.
+2026-01-07 12:04:41,848 - easytorch-training - INFO - Result <val>: [val/time: 6.43 (s), val/loss: 3.2007]
+2026-01-07 12:04:41,977 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 12:04:42,095 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_030000.pt saved
+2026-01-07 12:04:42,096 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 17:20:39
+2026-01-07 12:52:50,770 - easytorch-training - INFO - Iteration 35000 / 200003
+2026-01-07 12:52:51,138 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 3.86e-04, train/loss: 1.7605, train/grad_norm: 3.1830, train/amp_scale: 1.0000]
+2026-01-07 12:52:51,139 - easytorch-training - INFO - Start validation.
+2026-01-07 12:52:57,665 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.1925]
+2026-01-07 12:52:57,783 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 12:52:57,890 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_035000.pt saved
+2026-01-07 12:52:57,891 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 17:01:19
+2026-01-07 13:41:36,580 - easytorch-training - INFO - Iteration 40000 / 200003
+2026-01-07 13:41:36,948 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 3.80e-04, train/loss: 1.7497, train/grad_norm: 3.1247, train/amp_scale: 1.0000]
+2026-01-07 13:41:36,949 - easytorch-training - INFO - Start validation.
+2026-01-07 13:41:43,464 - easytorch-training - INFO - Result <val>: [val/time: 6.39 (s), val/loss: 3.1783]
+2026-01-07 13:41:43,580 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 13:41:43,692 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_040000.pt saved
+2026-01-07 13:41:43,692 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:49:20
+2026-01-07 14:30:28,198 - easytorch-training - INFO - Iteration 45000 / 200003
+2026-01-07 14:30:28,571 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 3.72e-04, train/loss: 1.7527, train/grad_norm: 3.1179, train/amp_scale: 1.0000]
+2026-01-07 14:30:28,571 - easytorch-training - INFO - Start validation.
+2026-01-07 14:30:35,095 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.1884]
+2026-01-07 14:30:35,203 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_045000.pt saved
+2026-01-07 14:30:35,203 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:40:26
+2026-01-07 15:19:32,332 - easytorch-training - INFO - Iteration 50000 / 200003
+2026-01-07 15:19:32,703 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 3.63e-04, train/loss: 1.7219, train/grad_norm: 3.1227, train/amp_scale: 1.0000]
+2026-01-07 15:19:32,704 - easytorch-training - INFO - Start validation.
+2026-01-07 15:19:39,229 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.1710]
+2026-01-07 15:19:39,354 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 15:19:39,471 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_050000.pt saved
+2026-01-07 15:19:39,473 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:34:09
+2026-01-07 16:08:29,693 - easytorch-training - INFO - Iteration 55000 / 200003
+2026-01-07 16:08:30,063 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 3.53e-04, train/loss: 1.6934, train/grad_norm: 3.1966, train/amp_scale: 1.0000]
+2026-01-07 16:08:30,063 - easytorch-training - INFO - Start validation.
+2026-01-07 16:08:36,589 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 3.1712]
+2026-01-07 16:08:36,708 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_055000.pt saved
+2026-01-07 16:08:36,709 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:28:36
+2026-01-07 16:57:25,056 - easytorch-training - INFO - Iteration 60000 / 200003
+2026-01-07 16:57:25,429 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 3.41e-04, train/loss: 1.6860, train/grad_norm: 3.1026, train/amp_scale: 1.0000]
+2026-01-07 16:57:25,430 - easytorch-training - INFO - Start validation.
+2026-01-07 16:57:31,953 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.1621]
+2026-01-07 16:57:32,079 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 16:57:32,196 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_060000.pt saved
+2026-01-07 16:57:32,197 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:23:52
+2026-01-07 17:46:40,426 - easytorch-training - INFO - Iteration 65000 / 200003
+2026-01-07 17:46:40,798 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 3.29e-04, train/loss: 1.6761, train/grad_norm: 3.0234, train/amp_scale: 1.0000]
+2026-01-07 17:46:40,798 - easytorch-training - INFO - Start validation.
+2026-01-07 17:46:47,379 - easytorch-training - INFO - Result <val>: [val/time: 6.46 (s), val/loss: 3.1677]
+2026-01-07 17:46:47,498 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_065000.pt saved
+2026-01-07 17:46:47,498 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:20:53
+2026-01-07 18:34:08,164 - easytorch-training - INFO - Iteration 70000 / 200003
+2026-01-07 18:34:08,534 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.57 (s), train/lr: 3.16e-04, train/loss: 1.6859, train/grad_norm: 3.1570, train/amp_scale: 1.0000]
+2026-01-07 18:34:08,535 - easytorch-training - INFO - Start validation.
+2026-01-07 18:34:15,064 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 3.1601]
+2026-01-07 18:34:15,188 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 18:34:15,304 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_070000.pt saved
+2026-01-07 18:34:15,304 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:13:12
+2026-01-07 19:21:47,633 - easytorch-training - INFO - Iteration 75000 / 200003
+2026-01-07 19:21:48,003 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.57 (s), train/lr: 3.02e-04, train/loss: 1.6886, train/grad_norm: 3.0815, train/amp_scale: 1.0000]
+2026-01-07 19:21:48,004 - easytorch-training - INFO - Start validation.
+2026-01-07 19:21:54,547 - easytorch-training - INFO - Result <val>: [val/time: 6.42 (s), val/loss: 3.1612]
+2026-01-07 19:21:54,657 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_075000.pt saved
+2026-01-07 19:21:54,658 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:07:04
+2026-01-07 20:10:41,673 - easytorch-training - INFO - Iteration 80000 / 200003
+2026-01-07 20:10:42,120 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 2.88e-04, train/loss: 1.6720, train/grad_norm: 3.0019, train/amp_scale: 1.0000]
+2026-01-07 20:10:42,120 - easytorch-training - INFO - Start validation.
+2026-01-07 20:10:49,850 - easytorch-training - INFO - Result <val>: [val/time: 7.61 (s), val/loss: 3.1598]
+2026-01-07 20:10:49,968 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 20:10:50,076 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_080000.pt saved
+2026-01-07 20:10:50,077 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:04:52
+2026-01-07 20:59:31,371 - easytorch-training - INFO - Iteration 85000 / 200003
+2026-01-07 20:59:31,737 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 2.73e-04, train/loss: 1.6886, train/grad_norm: 3.0279, train/amp_scale: 1.0000]
+2026-01-07 20:59:31,738 - easytorch-training - INFO - Start validation.
+2026-01-07 20:59:38,254 - easytorch-training - INFO - Result <val>: [val/time: 6.39 (s), val/loss: 3.1565]
+2026-01-07 20:59:38,380 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 20:59:38,498 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_085000.pt saved
+2026-01-07 20:59:38,499 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 16:02:38
+2026-01-07 21:46:32,409 - easytorch-training - INFO - Iteration 90000 / 200003
+2026-01-07 21:46:32,778 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.56 (s), train/lr: 2.57e-04, train/loss: 1.6757, train/grad_norm: 2.9444, train/amp_scale: 1.0000]
+2026-01-07 21:46:32,778 - easytorch-training - INFO - Start validation.
+2026-01-07 21:46:39,315 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 3.1490]
+2026-01-07 21:46:39,444 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 21:46:39,562 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_090000.pt saved
+2026-01-07 21:46:39,563 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:56:41
+2026-01-07 22:35:28,461 - easytorch-training - INFO - Iteration 95000 / 200003
+2026-01-07 22:35:28,830 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 2.41e-04, train/loss: 1.6787, train/grad_norm: 3.0766, train/amp_scale: 1.0000]
+2026-01-07 22:35:28,830 - easytorch-training - INFO - Start validation.
+2026-01-07 22:35:35,488 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.1522]
+2026-01-07 22:35:35,596 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_095000.pt saved
+2026-01-07 22:35:35,597 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:55:24
+2026-01-07 23:23:36,357 - easytorch-training - INFO - Iteration 100000 / 200003
+2026-01-07 23:23:36,728 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 2.25e-04, train/loss: 1.6803, train/grad_norm: 2.9411, train/amp_scale: 1.0000]
+2026-01-07 23:23:36,729 - easytorch-training - INFO - Start validation.
+2026-01-07 23:23:43,275 - easytorch-training - INFO - Result <val>: [val/time: 6.42 (s), val/loss: 3.1480]
+2026-01-07 23:23:43,392 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-07 23:23:43,502 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_100000.pt saved
+2026-01-07 23:23:43,503 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:52:38
+2026-01-08 00:12:26,912 - easytorch-training - INFO - Iteration 105000 / 200003
+2026-01-08 00:12:27,283 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 2.08e-04, train/loss: 1.6578, train/grad_norm: 2.9696, train/amp_scale: 1.0000]
+2026-01-08 00:12:27,284 - easytorch-training - INFO - Start validation.
+2026-01-08 00:12:33,832 - easytorch-training - INFO - Result <val>: [val/time: 6.43 (s), val/loss: 3.1484]
+2026-01-08 00:12:33,946 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_105000.pt saved
+2026-01-08 00:12:33,947 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:51:29
+2026-01-08 01:00:28,211 - easytorch-training - INFO - Iteration 110000 / 200003
+2026-01-08 01:00:28,578 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.57 (s), train/lr: 1.92e-04, train/loss: 1.6578, train/grad_norm: 2.9220, train/amp_scale: 1.0000]
+2026-01-08 01:00:28,578 - easytorch-training - INFO - Start validation.
+2026-01-08 01:00:35,509 - easytorch-training - INFO - Result <val>: [val/time: 6.81 (s), val/loss: 3.1463]
+2026-01-08 01:00:35,625 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-08 01:00:35,733 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_110000.pt saved
+2026-01-08 01:00:35,733 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:48:58
+2026-01-08 01:48:57,594 - easytorch-training - INFO - Iteration 115000 / 200003
+2026-01-08 01:48:57,963 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 1.75e-04, train/loss: 1.6509, train/grad_norm: 2.9594, train/amp_scale: 1.0000]
+2026-01-08 01:48:57,963 - easytorch-training - INFO - Start validation.
+2026-01-08 01:49:04,521 - easytorch-training - INFO - Result <val>: [val/time: 6.44 (s), val/loss: 3.1445]
+2026-01-08 01:49:04,637 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_best_val_loss.pt saved
+2026-01-08 01:49:04,745 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_115000.pt saved
+2026-01-08 01:49:04,746 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:47:27
+2026-01-08 02:37:42,819 - easytorch-training - INFO - Iteration 120000 / 200003
+2026-01-08 02:37:43,186 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 1.59e-04, train/loss: 1.6437, train/grad_norm: 3.2303, train/amp_scale: 1.0000]
+2026-01-08 02:37:43,187 - easytorch-training - INFO - Start validation.
+2026-01-08 02:37:49,711 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 3.1487]
+2026-01-08 02:37:49,820 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_120000.pt saved
+2026-01-08 02:37:49,821 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:46:31
+2026-01-08 03:27:03,918 - easytorch-training - INFO - Iteration 125000 / 200003
+2026-01-08 03:27:04,286 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 1.43e-04, train/loss: 1.8331, train/grad_norm: 57.6050, train/amp_scale: 1.0000]
+2026-01-08 03:27:04,287 - easytorch-training - INFO - Start validation.
+2026-01-08 03:27:10,815 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 3.3145]
+2026-01-08 03:27:10,922 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_125000.pt saved
+2026-01-08 03:27:10,923 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:46:37
+2026-01-08 04:16:06,610 - easytorch-training - INFO - Iteration 130000 / 200003
+2026-01-08 04:16:06,980 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.59 (s), train/lr: 1.27e-04, train/loss: 2.4557, train/grad_norm: 1590.0738, train/amp_scale: 1.0000]
+2026-01-08 04:16:06,980 - easytorch-training - INFO - Start validation.
+2026-01-08 04:16:13,506 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 4.2894]
+2026-01-08 04:16:13,613 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_130000.pt saved
+2026-01-08 04:16:13,614 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:46:14
+2026-01-08 05:04:55,295 - easytorch-training - INFO - Iteration 135000 / 200003
+2026-01-08 05:04:55,663 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 1.12e-04, train/loss: 3.9402, train/grad_norm: 11837.9622, train/amp_scale: 1.0000]
+2026-01-08 05:04:55,664 - easytorch-training - INFO - Start validation.
+2026-01-08 05:05:02,197 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 4.5390]
+2026-01-08 05:05:02,306 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_135000.pt saved
+2026-01-08 05:05:02,307 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:45:32
+2026-01-08 05:53:41,209 - easytorch-training - INFO - Iteration 140000 / 200003
+2026-01-08 05:53:41,574 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 9.77e-05, train/loss: 4.2236, train/grad_norm: 497822.2043, train/amp_scale: 1.0000]
+2026-01-08 05:53:41,574 - easytorch-training - INFO - Start validation.
+2026-01-08 05:53:48,090 - easytorch-training - INFO - Result <val>: [val/time: 6.39 (s), val/loss: 4.3929]
+2026-01-08 05:53:48,197 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_140000.pt saved
+2026-01-08 05:53:48,198 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:44:49
+2026-01-08 06:42:18,216 - easytorch-training - INFO - Iteration 145000 / 200003
+2026-01-08 06:42:18,586 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 8.38e-05, train/loss: 4.6652, train/grad_norm: 261043219.4423, train/amp_scale: 1.0000]
+2026-01-08 06:42:18,587 - easytorch-training - INFO - Start validation.
+2026-01-08 06:42:25,113 - easytorch-training - INFO - Result <val>: [val/time: 6.40 (s), val/loss: 4.6455]
+2026-01-08 06:42:25,225 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_145000.pt saved
+2026-01-08 06:42:25,226 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:43:56
+2026-01-08 07:30:49,543 - easytorch-training - INFO - Iteration 150000 / 200003
+2026-01-08 07:30:49,926 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 7.08e-05, train/loss: 4.8012, train/grad_norm: 542310146.5395, train/amp_scale: 1.0000]
+2026-01-08 07:30:49,926 - easytorch-training - INFO - Start validation.
+2026-01-08 07:30:56,443 - easytorch-training - INFO - Result <val>: [val/time: 6.39 (s), val/loss: 4.5128]
+2026-01-08 07:30:56,554 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_150000.pt saved
+2026-01-08 07:30:56,555 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:43:00
+2026-01-08 08:19:05,663 - easytorch-training - INFO - Iteration 155000 / 200003
+2026-01-08 08:19:06,031 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.58 (s), train/lr: 5.86e-05, train/loss: 4.5055, train/grad_norm: 370485325.2526, train/amp_scale: 1.0000]
+2026-01-08 08:19:06,031 - easytorch-training - INFO - Start validation.
+2026-01-08 08:19:12,566 - easytorch-training - INFO - Result <val>: [val/time: 6.41 (s), val/loss: 4.8368]
+2026-01-08 08:19:12,674 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200003/937fc7f323a4d7d3ae9480105345ef67/TimeMoE4_155000.pt saved
+2026-01-08 08:19:12,675 - easytorch-training - INFO - The estimated training finish time is 2026-01-08 15:41:48
+2026-01-08 08:30:31,699 - easytorch-training - ERROR - Traceback (most recent call last):
+  File "/home/nvidia/miniconda3/envs/zxx/lib/python3.11/site-packages/easytorch/launcher/launcher.py", line 31, in training_func
+    runner.train(cfg)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_iteration_runner.py", line 200, in train
+    self.train_iters(iteration=iteration, dataloader=self.train_data_loader)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 281, in train_iters
+    self.backward(loss, accumulating=accumulating)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 337, in backward
+    grad_norm = sum(
+                ^^^^
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 338, in <genexpr>
+    param.grad.data.norm(2).item() ** 2 for param in self.model.parameters() if param.grad is not None
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+