Upload Main_200020

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_005000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04293ce93939e93ba8bec6deddc5db514e769d10590f868464879ecb210a1a3
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_010000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:771971660cf8b81dc30906c6d61d0d1e88f1865fd4bda3f0c297970c3d356998
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_015000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab77468cbf8b1a0da7f10874215213c248f2c8ebc1f7ba8bbc77a59d9d10ad2b
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_020000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4946cd8fed06135edc181c58952b33fb5837e7264c97c951818762c421cff3d1
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_025000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11f8ad82ab91a5df6caa102d83c5a0450552e9ed124591d2d12887d6c857e72e
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_030000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45473e4e3b19c70fcddb0d722b6e917c31af6da438559d8d3554f4975e012bb3
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_035000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5229b46cb04fa64ab45bc928efb5ac7decbae478642c394c1615df7772929833
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_040000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70f8c6c4d7a2adea3b0ea21d4e70703b743bc57d51283edb135720fb3dd4ef85
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_045000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:170c1867c6dc1d9637eb7b608547cb3bc31ff8a702ceac8c02b1403846d7b174
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_050000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c61c3a308461031b6dc7f033408339389cc53085f3440892f115075d34cd32c
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_055000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f558c4c911ee70b4cd2e17a6cec120b5d0b715aae1d7435153615dc12b04bdd
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_060000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd99e7ff866cac4464806bfd818339a6177b515593b6e29049745f7699550dd5
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_065000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e5b46c762c7fcaccb748af9e82fe7e7ac2069503b5e4d0a2c9d0a3a8e9634e1
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_070000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0a1af376c2c0f7282c7d6609849f99d780e2d587536ed31470b70d63d0c7f73
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_075000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84979e577388c04e0bac8eae074da2615cfcd7f53b6de41cda579bbd7db487f5
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_080000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b0cb7ec422b85c8670204f822fc44da062b9069e2d612fc607d1ccada936132
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_085000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be8a21bf230669387002f21cf317e0b7ef8692fb36daa99e56ddbddb2cee4a88
+size 151982040

Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b7def102a3cc573c2b1e3dde7f42eaac6ae48ea80a121b3cb9e30b75ec90420
+size 151985051

Main_200020/911b09031e845e188707162e012110cb/cfg.txt

@@ -0,0 +1,55 @@
+DESCRIPTION: TimeMoE Base
+DEVICE: gpu
+DEVICE_NUM: 3
+RUNNER: <class 'baselines.TimeMoE4.runner.runner.TimeMoERunner'>
+MODEL:
+  NAME: TimeMoE4
+  ARCH: <class 'baselines.TimeMoE4.arch.timemoe.TimeMoE4'>
+  PARAM:
+    model_id: baselines/TimeMoE/ckpt/TimeMoE-50M
+    from_pretrained: False
+    context_length: 4079
+    trust_remote_code: True
+  DTYPE: bfloat16
+METRICS:
+  FUNCS:
+TRAIN:
+  COMPILE_MODEL: True
+  NUM_ITERATIONS: 200020
+  CKPT_SAVE_DIR: checkpoints/TimeMoE4/Main_200020
+  CKPT_SAVE_STRATEGY: 5000
+  LOSS: fake_loss
+  OPTIM:
+    TYPE: AdamW
+    PARAM:
+      lr: 0.001
+      betas: (0.9, 0.95)
+      fused: True
+  LR_SCHEDULER:
+    TYPE: CosineWarmup
+    PARAM:
+      num_warmup_steps: 10000
+      num_training_steps: 200020
+  CLIP_GRAD_PARAM:
+    max_norm: 1.0
+  DATA:
+    BATCH_SIZE: 85
+    SHUFFLE: True
+    PIN_MEMORY: True
+    PREFETCH: True
+  GRAD_ACCUMULATION_STEPS: 1
+VAL:
+  INTERVAL: 5000
+  DATA:
+    BATCH_SIZE: 170
+EVAL:
+  USE_GPU: True
+DATASET:
+  NAME: Main
+  TYPE: <class 'baselines.TimeMoE4.data.mix_dataset_v2.MixedSourceDataset_v2'>
+  PARAM:
+    num_valid_samples: 1000
+INFERENCE:
+  GENERATION_PARAMS:
+    normalize: True
+MD5: 911b09031e845e188707162e012110cb

Main_200020/911b09031e845e188707162e012110cb/tensorboard/events.out.tfevents.1769154948.brev-5x9knwe1p.203240.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:652793da37d36ddfef06c169e64f3660397dd29135b720559ddac341c0f69c76
+size 236220

Main_200020/911b09031e845e188707162e012110cb/tensorboard/events.out.tfevents.1769165586.brev-5x9knwe1p.470619.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b8d260e587b97b930aaa496b48de763bc036fd76b405ef4a65baec82e2597b0
+size 18209918

Main_200020/911b09031e845e188707162e012110cb/timemoe_base.py

@@ -0,0 +1,126 @@
+# 采样概率变化
+
+import os
+import sys
+from easydict import EasyDict
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ..arch import TimeMoE4
+from ..data import MixedSourceDataset_v2
+from ..runner import TimeMoERunner
+from ..loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+pretrained = False  # Whether to use a pretrained model
+
+MODEL_ARCH = TimeMoE4
+
+MODEL_PARAM = {
+    'model_id': "baselines/TimeMoE/ckpt/TimeMoE-50M",
+    'from_pretrained': pretrained,
+    'context_length': 4079,
+    'trust_remote_code': True,
+}
+DATA_NAME = "Main"
+
+# N = 20_000_000
+# batch size = 16*8
+# 20_000_000 / 16 / 8 = 156250 iterations
+# 20_000_000 * 4096 / 16 / 8 / 4096 = 156_250
+
+NUM_ITERATIONS = 200_020 # 总轮数   20_000_000 * 4096 / 16 / 4 / 4096 = 312,500
+VAL_ITERATION_INTERVAL = 5_000 # 每VAL_ITERATION_INTERVAL执行一次验证
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'TimeMoE Base'
+CFG.DEVICE = 'gpu'
+CFG.DEVICE_NUM = 3
+# Runner
+CFG.RUNNER = TimeMoERunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+# CFG.MODEL.DTYPE= 'float32'
+
+############################## Metrics Configuration ##############################
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({})
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = True
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+CFG.TRAIN.CKPT_SAVE_STRATEGY = VAL_ITERATION_INTERVAL * 1 # 保存策略，每VAL_ITERATION_INTERVAL * 5保存一次模型
+CFG.TRAIN.LOSS = fake_loss
+# Optimizer settings
+CFG.TRAIN.OPTIM = EasyDict()
+CFG.TRAIN.OPTIM.TYPE = "AdamW"
+CFG.TRAIN.OPTIM.PARAM = {
+    "lr": 1e-3,
+    "betas": (0.9, 0.95),
+    # "betas": (0.9, 0.98),
+    "fused": True,
+    # "weight_decay": 1e-1,
+}
+# Learning rate scheduler settings
+CFG.TRAIN.LR_SCHEDULER = EasyDict()
+CFG.TRAIN.LR_SCHEDULER.TYPE = "CosineWarmup"
+CFG.TRAIN.LR_SCHEDULER.PARAM = {
+    'num_warmup_steps': 10_000, # 10k
+    'num_training_steps': NUM_ITERATIONS,
+}
+CFG.TRAIN.CLIP_GRAD_PARAM = {
+    'max_norm': 1.0
+}
+# Train data loader settings
+CFG.TRAIN.DATA = EasyDict()
+CFG.TRAIN.DATA.BATCH_SIZE = 85  # 16  /  4
+CFG.TRAIN.DATA.SHUFFLE = True # has to be False
+CFG.TRAIN.DATA.PIN_MEMORY = True
+CFG.TRAIN.DATA.PREFETCH = True
+CFG.TRAIN.GRAD_ACCUMULATION_STEPS = 1
+# CFG.TRAIN.DATA.NUM_WORKERS = 4
+
+############################## Validation Configuration ##############################
+CFG.VAL = EasyDict()
+CFG.VAL.INTERVAL = VAL_ITERATION_INTERVAL
+CFG.VAL.DATA = EasyDict()
+CFG.VAL.DATA.BATCH_SIZE = 170  # 32  /  8
+
+############################## Evaluation Configuration ##############################
+
+CFG.EVAL = EasyDict()
+# Evaluation parameters
+CFG.EVAL.USE_GPU = True # Whether to use GPU for evaluation. Default: True
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = MixedSourceDataset_v2
+CFG.DATASET.PARAM = EasyDict({
+    'num_valid_samples': 1000
+})
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+    'normalize': not pretrained
+})
+

Main_200020/911b09031e845e188707162e012110cb/training_log_20260123075538.log

@@ -0,0 +1,45 @@
+2026-01-23 07:55:38,389 - easytorch-training - INFO - Initializing training.
+2026-01-23 07:55:38,389 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-23 07:55:38,390 - easytorch-training - INFO - Building training data loader.
+2026-01-23 07:55:48,766 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-23 07:55:48,766 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-23 07:55:48,766 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-23 07:55:48,767 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-23 07:55:48,768 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.001
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-23 07:55:48,769 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x7b8d7429e190>
+2026-01-23 07:55:48,770 - easytorch-training - INFO - Initializing validation.
+2026-01-23 07:55:48,770 - easytorch-training - INFO - Building val data loader.
+2026-01-23 07:55:49,068 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-23 07:56:15,319 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-23 07:56:15,319 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-23 07:56:15,320 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-23 07:56:15,320 - easytorch-training - INFO - Number of parameters: 12653568
+2026-01-23 07:56:15,321 - easytorch-training - INFO - Training with 3 GPUs, batch size per GPUs: 85, grad_accumulation_steps: 1
+2026-01-23 07:56:15,321 - easytorch-training - INFO - Effective batch size: 255
+2026-01-23 08:09:17,357 - easytorch-training - ERROR - Traceback (most recent call last):
+  File "/home/nvidia/miniconda3/envs/zxx/lib/python3.11/site-packages/easytorch/launcher/launcher.py", line 31, in training_func
+    runner.train(cfg)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_iteration_runner.py", line 200, in train
+    self.train_iters(iteration=iteration, dataloader=self.train_data_loader)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 281, in train_iters
+    self.backward(loss, accumulating=accumulating)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 337, in backward
+    grad_norm = sum(
+                ^^^^
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 338, in <genexpr>
+    param.grad.data.norm(2).item() ** 2 for param in self.model.parameters() if param.grad is not None
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+

Main_200020/911b09031e845e188707162e012110cb/training_log_20260123105255.log

@@ -0,0 +1,157 @@
+2026-01-23 10:52:55,325 - easytorch-training - INFO - Initializing training.
+2026-01-23 10:52:55,326 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-23 10:52:55,326 - easytorch-training - INFO - Building training data loader.
+2026-01-23 10:53:06,177 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-23 10:53:06,178 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-23 10:53:06,178 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-23 10:53:06,178 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-23 10:53:06,180 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.001
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-23 10:53:06,181 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x7e03180e6410>
+2026-01-23 10:53:06,182 - easytorch-training - INFO - Initializing validation.
+2026-01-23 10:53:06,182 - easytorch-training - INFO - Building val data loader.
+2026-01-23 10:53:08,158 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-23 10:53:32,319 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-23 10:53:32,319 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-23 10:53:32,319 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-23 10:53:32,320 - easytorch-training - INFO - Number of parameters: 12653568
+2026-01-23 10:53:32,320 - easytorch-training - INFO - Training with 3 GPUs, batch size per GPUs: 85, grad_accumulation_steps: 1
+2026-01-23 10:53:32,320 - easytorch-training - INFO - Effective batch size: 255
+2026-01-23 11:51:44,511 - easytorch-training - INFO - Iteration 5000 / 200020
+2026-01-23 11:51:45,002 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 2.50e-04, train/loss: 3.2199, train/grad_norm: 5.9624, train/amp_scale: 1.0000]
+2026-01-23 11:51:45,002 - easytorch-training - INFO - Start validation.
+2026-01-23 11:52:00,020 - easytorch-training - INFO - Result <val>: [val/time: 14.84 (s), val/loss: 3.3060]
+2026-01-23 11:52:00,146 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 11:52:00,263 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_005000.pt saved
+2026-01-23 11:52:00,264 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 01:52:24
+2026-01-23 12:51:15,946 - easytorch-training - INFO - Iteration 10000 / 200020
+2026-01-23 12:51:16,434 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 7.50e-04, train/loss: 2.3656, train/grad_norm: 2.8994, train/amp_scale: 1.0000]
+2026-01-23 12:51:16,434 - easytorch-training - INFO - Start validation.
+2026-01-23 12:51:23,127 - easytorch-training - INFO - Result <val>: [val/time: 6.51 (s), val/loss: 3.2728]
+2026-01-23 12:51:23,261 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 12:51:23,384 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_010000.pt saved
+2026-01-23 12:51:23,385 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:10:47
+2026-01-23 13:51:43,664 - easytorch-training - INFO - Iteration 15000 / 200020
+2026-01-23 13:51:44,153 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 9.99e-04, train/loss: 2.2469, train/grad_norm: 2.5323, train/amp_scale: 1.0000]
+2026-01-23 13:51:44,154 - easytorch-training - INFO - Start validation.
+2026-01-23 13:51:50,859 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.2238]
+2026-01-23 13:51:50,991 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 13:51:51,114 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_015000.pt saved
+2026-01-23 13:51:51,115 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:31:17
+2026-01-23 14:51:26,781 - easytorch-training - INFO - Iteration 20000 / 200020
+2026-01-23 14:51:27,270 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 9.96e-04, train/loss: 2.1872, train/grad_norm: 2.5239, train/amp_scale: 1.0000]
+2026-01-23 14:51:27,270 - easytorch-training - INFO - Start validation.
+2026-01-23 14:51:33,971 - easytorch-training - INFO - Result <val>: [val/time: 6.52 (s), val/loss: 3.2241]
+2026-01-23 14:51:34,096 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_020000.pt saved
+2026-01-23 14:51:34,097 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:34:04
+2026-01-23 15:51:38,857 - easytorch-training - INFO - Iteration 25000 / 200020
+2026-01-23 15:51:39,344 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 9.89e-04, train/loss: 2.1625, train/grad_norm: 2.5963, train/amp_scale: 1.0000]
+2026-01-23 15:51:39,345 - easytorch-training - INFO - Start validation.
+2026-01-23 15:51:46,055 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.2003]
+2026-01-23 15:51:46,188 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 15:51:46,309 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_025000.pt saved
+2026-01-23 15:51:46,310 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:39:38
+2026-01-23 16:51:22,311 - easytorch-training - INFO - Iteration 30000 / 200020
+2026-01-23 16:51:22,798 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 9.79e-04, train/loss: 2.1411, train/grad_norm: 2.5767, train/amp_scale: 1.0000]
+2026-01-23 16:51:22,799 - easytorch-training - INFO - Start validation.
+2026-01-23 16:51:29,495 - easytorch-training - INFO - Result <val>: [val/time: 6.52 (s), val/loss: 3.1886]
+2026-01-23 16:51:29,626 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 16:51:29,747 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_030000.pt saved
+2026-01-23 16:51:29,748 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:40:09
+2026-01-23 17:51:16,104 - easytorch-training - INFO - Iteration 35000 / 200020
+2026-01-23 17:51:16,594 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 9.66e-04, train/loss: 2.1309, train/grad_norm: 2.5103, train/amp_scale: 1.0000]
+2026-01-23 17:51:16,595 - easytorch-training - INFO - Start validation.
+2026-01-23 17:51:23,293 - easytorch-training - INFO - Result <val>: [val/time: 6.52 (s), val/loss: 3.1921]
+2026-01-23 17:51:23,415 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_035000.pt saved
+2026-01-23 17:51:23,415 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:41:30
+2026-01-23 18:51:00,864 - easytorch-training - INFO - Iteration 40000 / 200020
+2026-01-23 18:51:01,359 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 9.49e-04, train/loss: 2.1304, train/grad_norm: 2.5564, train/amp_scale: 1.0000]
+2026-01-23 18:51:01,359 - easytorch-training - INFO - Start validation.
+2026-01-23 18:51:08,076 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.1783]
+2026-01-23 18:51:08,195 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 18:51:08,303 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_040000.pt saved
+2026-01-23 18:51:08,304 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:41:46
+2026-01-23 19:49:52,036 - easytorch-training - INFO - Iteration 45000 / 200020
+2026-01-23 19:49:52,526 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 9.29e-04, train/loss: 2.1211, train/grad_norm: 2.5939, train/amp_scale: 1.0000]
+2026-01-23 19:49:52,527 - easytorch-training - INFO - Start validation.
+2026-01-23 19:49:59,615 - easytorch-training - INFO - Result <val>: [val/time: 6.91 (s), val/loss: 3.1864]
+2026-01-23 19:49:59,737 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_045000.pt saved
+2026-01-23 19:49:59,738 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:38:01
+2026-01-23 20:50:51,802 - easytorch-training - INFO - Iteration 50000 / 200020
+2026-01-23 20:50:52,288 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.73 (s), train/lr: 9.07e-04, train/loss: 2.1259, train/grad_norm: 2.6198, train/amp_scale: 1.0000]
+2026-01-23 20:50:52,288 - easytorch-training - INFO - Start validation.
+2026-01-23 20:50:58,952 - easytorch-training - INFO - Result <val>: [val/time: 6.49 (s), val/loss: 3.1789]
+2026-01-23 20:50:59,069 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_050000.pt saved
+2026-01-23 20:50:59,070 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:43:33
+2026-01-23 21:51:10,960 - easytorch-training - INFO - Iteration 55000 / 200020
+2026-01-23 21:51:11,451 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 8.81e-04, train/loss: 2.1204, train/grad_norm: 2.6293, train/amp_scale: 1.0000]
+2026-01-23 21:51:11,452 - easytorch-training - INFO - Start validation.
+2026-01-23 21:51:18,184 - easytorch-training - INFO - Result <val>: [val/time: 6.56 (s), val/loss: 3.1749]
+2026-01-23 21:51:18,322 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 21:51:18,441 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_055000.pt saved
+2026-01-23 21:51:18,443 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:45:39
+2026-01-23 22:52:01,528 - easytorch-training - INFO - Iteration 60000 / 200020
+2026-01-23 22:52:02,018 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.73 (s), train/lr: 8.53e-04, train/loss: 2.1224, train/grad_norm: 2.5976, train/amp_scale: 1.0000]
+2026-01-23 22:52:02,020 - easytorch-training - INFO - Start validation.
+2026-01-23 22:52:08,700 - easytorch-training - INFO - Result <val>: [val/time: 6.50 (s), val/loss: 3.1780]
+2026-01-23 22:52:08,826 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_060000.pt saved
+2026-01-23 22:52:08,827 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:49:08
+2026-01-23 23:50:20,226 - easytorch-training - INFO - Iteration 65000 / 200020
+2026-01-23 23:50:20,713 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 8.23e-04, train/loss: 2.1122, train/grad_norm: 2.6192, train/amp_scale: 1.0000]
+2026-01-23 23:50:20,714 - easytorch-training - INFO - Start validation.
+2026-01-23 23:50:27,777 - easytorch-training - INFO - Result <val>: [val/time: 6.88 (s), val/loss: 3.1671]
+2026-01-23 23:50:27,907 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-23 23:50:28,027 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_065000.pt saved
+2026-01-23 23:50:28,028 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:44:19
+2026-01-24 00:50:27,464 - easytorch-training - INFO - Iteration 70000 / 200020
+2026-01-24 00:50:27,948 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 7.90e-04, train/loss: 2.1068, train/grad_norm: 2.6063, train/amp_scale: 1.0000]
+2026-01-24 00:50:27,949 - easytorch-training - INFO - Start validation.
+2026-01-24 00:50:34,624 - easytorch-training - INFO - Result <val>: [val/time: 6.49 (s), val/loss: 3.1752]
+2026-01-24 00:50:34,733 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_070000.pt saved
+2026-01-24 00:50:34,734 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:45:19
+2026-01-24 01:50:38,702 - easytorch-training - INFO - Iteration 75000 / 200020
+2026-01-24 01:50:39,191 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 7.56e-04, train/loss: 2.1049, train/grad_norm: 2.6683, train/amp_scale: 1.0000]
+2026-01-24 01:50:39,191 - easytorch-training - INFO - Start validation.
+2026-01-24 01:50:46,665 - easytorch-training - INFO - Result <val>: [val/time: 7.29 (s), val/loss: 3.1668]
+2026-01-24 01:50:46,784 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-24 01:50:46,893 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_075000.pt saved
+2026-01-24 01:50:46,894 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:46:25
+2026-01-24 02:51:12,299 - easytorch-training - INFO - Iteration 80000 / 200020
+2026-01-24 02:51:12,786 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 7.20e-04, train/loss: 2.0962, train/grad_norm: 2.6452, train/amp_scale: 1.0000]
+2026-01-24 02:51:12,786 - easytorch-training - INFO - Start validation.
+2026-01-24 02:51:19,492 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.1590]
+2026-01-24 02:51:19,613 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_best_val_loss.pt saved
+2026-01-24 02:51:19,725 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_080000.pt saved
+2026-01-24 02:51:19,726 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:48:15
+2026-01-24 03:51:06,183 - easytorch-training - INFO - Iteration 85000 / 200020
+2026-01-24 03:51:06,667 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 6.82e-04, train/loss: 2.1013, train/grad_norm: 2.6316, train/amp_scale: 1.0000]
+2026-01-24 03:51:06,668 - easytorch-training - INFO - Start validation.
+2026-01-24 03:51:13,348 - easytorch-training - INFO - Result <val>: [val/time: 6.50 (s), val/loss: 3.1592]
+2026-01-24 03:51:13,468 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200020/911b09031e845e188707162e012110cb/TimeMoE4_085000.pt saved
+2026-01-24 03:51:13,469 - easytorch-training - INFO - The estimated training finish time is 2026-01-25 02:48:19
+2026-01-24 04:26:31,834 - easytorch-training - ERROR - Traceback (most recent call last):
+  File "/home/nvidia/miniconda3/envs/zxx/lib/python3.11/site-packages/easytorch/launcher/launcher.py", line 31, in training_func
+    runner.train(cfg)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_iteration_runner.py", line 200, in train
+    self.train_iters(iteration=iteration, dataloader=self.train_data_loader)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 281, in train_iters
+    self.backward(loss, accumulating=accumulating)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 337, in backward
+    grad_norm = sum(
+                ^^^^
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 338, in <genexpr>
+    param.grad.data.norm(2).item() ** 2 for param in self.model.parameters() if param.grad is not None
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+