Upload Main_200018

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_005000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b055ed485964dd7741a91478de3d666d5784444e905eec925e708bc7611c628e
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_010000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45294f9060bdcc0433996972f79a0884265da9fcefa18d6f101507f94cad5da1
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_015000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66f030c32bcb78d2acf5e9dc05efdd50397d7001ec8ada525dc2a74304187317
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_020000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:510313ae30e72fed0e966055a60e7b6663660186c8fadebe2c1ce37fb86c3f7a
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_025000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5be20c252bcfc0fe453c74692c31a8942b23d31c326d56a295617de3f945d35f
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_030000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4ba08df6d02ebff18b5304647183942c4012dc88205960f18de73ac3d734b7e
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_035000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebea5a9258288de570c5f3bc2ac63a86bb5cd9128bebcaa1f08fadf57a330ce7
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_040000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7c2aef7d8015a9f29b9597fbb427cf29ed442c5e9728ffc28606dd8d41173bf
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_045000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b38c7612f2a4409848d5de9e09121b1fce804595cb870a44ac9b8f09fe05fe
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_050000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac775094798867cf365b9f41803a370f510257e21b84e97d2f63095d7d2c5882
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_055000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4081d53371f745c00a3391ab933d05fd524ddc972ec7fa067b6d21346c90e7c
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_060000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca8143a722696641ebbd74c2f3858f01005cdac0715d6c9ecdcdd6304524f935
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_065000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922faafa533c8f44222d10cfaa7d431349d708de17e10b3a80ed445d1d6668c7
+size 151982040

Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b98768df656f42ce4aa9dcf307a49e494b2124276eeb3322be0f41180e379879
+size 151985051

Main_200018/acf6c6763f53a27cd3a36171d821d82b/cfg.txt

@@ -0,0 +1,55 @@
+DESCRIPTION: TimeMoE Base
+DEVICE: gpu
+DEVICE_NUM: 3
+RUNNER: <class 'baselines.TimeMoE4.runner.runner.TimeMoERunner'>
+MODEL:
+  NAME: TimeMoE4
+  ARCH: <class 'baselines.TimeMoE4.arch.timemoe.TimeMoE4'>
+  PARAM:
+    model_id: baselines/TimeMoE/ckpt/TimeMoE-50M
+    from_pretrained: False
+    context_length: 4079
+    trust_remote_code: True
+  DTYPE: bfloat16
+METRICS:
+  FUNCS:
+TRAIN:
+  COMPILE_MODEL: True
+  NUM_ITERATIONS: 200018
+  CKPT_SAVE_DIR: checkpoints/TimeMoE4/Main_200018
+  CKPT_SAVE_STRATEGY: 5000
+  LOSS: fake_loss
+  OPTIM:
+    TYPE: AdamW
+    PARAM:
+      lr: 0.001
+      betas: (0.9, 0.95)
+      fused: True
+  LR_SCHEDULER:
+    TYPE: CosineWarmup
+    PARAM:
+      num_warmup_steps: 10000
+      num_training_steps: 200018
+  CLIP_GRAD_PARAM:
+    max_norm: 1.0
+  DATA:
+    BATCH_SIZE: 85
+    SHUFFLE: True
+    PIN_MEMORY: True
+    PREFETCH: True
+  GRAD_ACCUMULATION_STEPS: 1
+VAL:
+  INTERVAL: 5000
+  DATA:
+    BATCH_SIZE: 170
+EVAL:
+  USE_GPU: True
+DATASET:
+  NAME: Main
+  TYPE: <class 'baselines.TimeMoE4.data.mix_dataset_v2.MixedSourceDataset_v2'>
+  PARAM:
+    num_valid_samples: 1000
+INFERENCE:
+  GENERATION_PARAMS:
+    normalize: True
+MD5: acf6c6763f53a27cd3a36171d821d82b

Main_200018/acf6c6763f53a27cd3a36171d821d82b/tensorboard/events.out.tfevents.1769020087.brev-5x9knwe1p.1065478.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:870c82056b4f0d6bcb55cd2f75d258c14f442be3c69cba996c35e46e17068299
+size 13634150

Main_200018/acf6c6763f53a27cd3a36171d821d82b/timemoe_base.py

@@ -0,0 +1,126 @@
+# 采样概率变化
+
+import os
+import sys
+from easydict import EasyDict
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ..arch import TimeMoE4
+from ..data import MixedSourceDataset_v2
+from ..runner import TimeMoERunner
+from ..loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+pretrained = False  # Whether to use a pretrained model
+
+MODEL_ARCH = TimeMoE4
+
+MODEL_PARAM = {
+    'model_id': "baselines/TimeMoE/ckpt/TimeMoE-50M",
+    'from_pretrained': pretrained,
+    'context_length': 4079,
+    'trust_remote_code': True,
+}
+DATA_NAME = "Main"
+
+# N = 20_000_000
+# batch size = 16*8
+# 20_000_000 / 16 / 8 = 156250 iterations
+# 20_000_000 * 4096 / 16 / 8 / 4096 = 156_250
+
+NUM_ITERATIONS = 200_018 # 总轮数   20_000_000 * 4096 / 16 / 4 / 4096 = 312,500
+VAL_ITERATION_INTERVAL = 5_000 # 每VAL_ITERATION_INTERVAL执行一次验证
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'TimeMoE Base'
+CFG.DEVICE = 'gpu'
+CFG.DEVICE_NUM = 3
+# Runner
+CFG.RUNNER = TimeMoERunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+# CFG.MODEL.DTYPE= 'float32'
+
+############################## Metrics Configuration ##############################
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({})
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = True
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+CFG.TRAIN.CKPT_SAVE_STRATEGY = VAL_ITERATION_INTERVAL * 1 # 保存策略，每VAL_ITERATION_INTERVAL * 5保存一次模型
+CFG.TRAIN.LOSS = fake_loss
+# Optimizer settings
+CFG.TRAIN.OPTIM = EasyDict()
+CFG.TRAIN.OPTIM.TYPE = "AdamW"
+CFG.TRAIN.OPTIM.PARAM = {
+    "lr": 1e-3,
+    "betas": (0.9, 0.95),
+    # "betas": (0.9, 0.98),
+    "fused": True,
+    # "weight_decay": 1e-1,
+}
+# Learning rate scheduler settings
+CFG.TRAIN.LR_SCHEDULER = EasyDict()
+CFG.TRAIN.LR_SCHEDULER.TYPE = "CosineWarmup"
+CFG.TRAIN.LR_SCHEDULER.PARAM = {
+    'num_warmup_steps': 10_000, # 10k
+    'num_training_steps': NUM_ITERATIONS,
+}
+CFG.TRAIN.CLIP_GRAD_PARAM = {
+    'max_norm': 1.0
+}
+# Train data loader settings
+CFG.TRAIN.DATA = EasyDict()
+CFG.TRAIN.DATA.BATCH_SIZE = 85  # 16  /  4
+CFG.TRAIN.DATA.SHUFFLE = True # has to be False
+CFG.TRAIN.DATA.PIN_MEMORY = True
+CFG.TRAIN.DATA.PREFETCH = True
+CFG.TRAIN.GRAD_ACCUMULATION_STEPS = 1
+# CFG.TRAIN.DATA.NUM_WORKERS = 4
+
+############################## Validation Configuration ##############################
+CFG.VAL = EasyDict()
+CFG.VAL.INTERVAL = VAL_ITERATION_INTERVAL
+CFG.VAL.DATA = EasyDict()
+CFG.VAL.DATA.BATCH_SIZE = 170  # 32  /  8
+
+############################## Evaluation Configuration ##############################
+
+CFG.EVAL = EasyDict()
+# Evaluation parameters
+CFG.EVAL.USE_GPU = True # Whether to use GPU for evaluation. Default: True
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = MixedSourceDataset_v2
+CFG.DATASET.PARAM = EasyDict({
+    'num_valid_samples': 1000
+})
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+    'normalize': not pretrained
+})
+

Main_200018/acf6c6763f53a27cd3a36171d821d82b/training_log_20260121182753.log

@@ -0,0 +1,133 @@
+2026-01-21 18:27:53,225 - easytorch-training - INFO - Initializing training.
+2026-01-21 18:27:53,225 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-21 18:27:53,226 - easytorch-training - INFO - Building training data loader.
+2026-01-21 18:28:07,863 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-21 18:28:07,864 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-21 18:28:07,864 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-21 18:28:07,864 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-21 18:28:07,867 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.001
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-21 18:28:07,867 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x7446b8174950>
+2026-01-21 18:28:07,868 - easytorch-training - INFO - Initializing validation.
+2026-01-21 18:28:07,869 - easytorch-training - INFO - Building val data loader.
+2026-01-21 18:28:11,063 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-21 18:28:34,427 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-21 18:28:34,427 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-21 18:28:34,427 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-21 18:28:34,428 - easytorch-training - INFO - Number of parameters: 12653568
+2026-01-21 18:28:34,428 - easytorch-training - INFO - Training with 3 GPUs, batch size per GPUs: 85, grad_accumulation_steps: 1
+2026-01-21 18:28:34,428 - easytorch-training - INFO - Effective batch size: 255
+2026-01-21 19:34:46,353 - easytorch-training - INFO - Iteration 5000 / 200018
+2026-01-21 19:34:46,845 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.79 (s), train/lr: 2.50e-04, train/loss: 3.5735, train/grad_norm: 6.2758, train/amp_scale: 1.0000]
+2026-01-21 19:34:46,846 - easytorch-training - INFO - Start validation.
+2026-01-21 19:35:01,878 - easytorch-training - INFO - Result <val>: [val/time: 14.85 (s), val/loss: 3.3217]
+2026-01-21 19:35:01,989 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-21 19:35:02,095 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_005000.pt saved
+2026-01-21 19:35:02,096 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 14:47:15
+2026-01-21 20:35:05,639 - easytorch-training - INFO - Iteration 10000 / 200018
+2026-01-21 20:35:06,130 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 7.50e-04, train/loss: 2.7588, train/grad_norm: 2.9374, train/amp_scale: 1.0000]
+2026-01-21 20:35:06,131 - easytorch-training - INFO - Start validation.
+2026-01-21 20:35:12,849 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.2390]
+2026-01-21 20:35:12,983 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-21 20:35:13,103 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_010000.pt saved
+2026-01-21 20:35:13,104 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 12:41:41
+2026-01-21 21:33:44,220 - easytorch-training - INFO - Iteration 15000 / 200018
+2026-01-21 21:33:44,711 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 9.99e-04, train/loss: 2.6467, train/grad_norm: 2.5058, train/amp_scale: 1.0000]
+2026-01-21 21:33:44,711 - easytorch-training - INFO - Start validation.
+2026-01-21 21:33:51,425 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.1982]
+2026-01-21 21:33:51,552 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-21 21:33:51,670 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_015000.pt saved
+2026-01-21 21:33:51,670 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 11:39:17
+2026-01-21 22:32:30,744 - easytorch-training - INFO - Iteration 20000 / 200018
+2026-01-21 22:32:31,228 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 9.96e-04, train/loss: 2.5970, train/grad_norm: 2.4983, train/amp_scale: 1.0000]
+2026-01-21 22:32:31,228 - easytorch-training - INFO - Start validation.
+2026-01-21 22:32:37,925 - easytorch-training - INFO - Result <val>: [val/time: 6.52 (s), val/loss: 3.1878]
+2026-01-21 22:32:38,053 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-21 22:32:38,171 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_020000.pt saved
+2026-01-21 22:32:38,173 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 11:09:25
+2026-01-21 23:31:00,094 - easytorch-training - INFO - Iteration 25000 / 200018
+2026-01-21 23:31:00,582 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 9.89e-04, train/loss: 2.5758, train/grad_norm: 2.4831, train/amp_scale: 1.0000]
+2026-01-21 23:31:00,582 - easytorch-training - INFO - Start validation.
+2026-01-21 23:31:07,290 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.1744]
+2026-01-21 23:31:07,419 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-21 23:31:07,537 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_025000.pt saved
+2026-01-21 23:31:07,537 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:49:12
+2026-01-22 00:27:46,491 - easytorch-training - INFO - Iteration 30000 / 200018
+2026-01-22 00:27:46,981 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.68 (s), train/lr: 9.79e-04, train/loss: 2.5595, train/grad_norm: 2.5564, train/amp_scale: 1.0000]
+2026-01-22 00:27:46,981 - easytorch-training - INFO - Start validation.
+2026-01-22 00:27:53,696 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.1661]
+2026-01-22 00:27:53,824 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-22 00:27:53,943 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_030000.pt saved
+2026-01-22 00:27:53,943 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:24:17
+2026-01-22 01:27:10,403 - easytorch-training - INFO - Iteration 35000 / 200018
+2026-01-22 01:27:10,888 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 9.66e-04, train/loss: 2.5426, train/grad_norm: 2.5068, train/amp_scale: 1.0000]
+2026-01-22 01:27:10,888 - easytorch-training - INFO - Start validation.
+2026-01-22 01:27:17,574 - easytorch-training - INFO - Result <val>: [val/time: 6.51 (s), val/loss: 3.1707]
+2026-01-22 01:27:17,703 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_035000.pt saved
+2026-01-22 01:27:17,704 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:21:28
+2026-01-22 02:27:38,931 - easytorch-training - INFO - Iteration 40000 / 200018
+2026-01-22 02:27:39,420 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 9.49e-04, train/loss: 2.5410, train/grad_norm: 2.5819, train/amp_scale: 1.0000]
+2026-01-22 02:27:39,420 - easytorch-training - INFO - Start validation.
+2026-01-22 02:27:46,137 - easytorch-training - INFO - Result <val>: [val/time: 6.54 (s), val/loss: 3.1605]
+2026-01-22 02:27:46,269 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-22 02:27:46,389 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_040000.pt saved
+2026-01-22 02:27:46,391 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:24:47
+2026-01-22 03:27:49,005 - easytorch-training - INFO - Iteration 45000 / 200018
+2026-01-22 03:27:49,495 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 9.29e-04, train/loss: 2.5352, train/grad_norm: 2.5853, train/amp_scale: 1.0000]
+2026-01-22 03:27:49,496 - easytorch-training - INFO - Start validation.
+2026-01-22 03:27:56,201 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.1562]
+2026-01-22 03:27:56,333 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-22 03:27:56,454 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_045000.pt saved
+2026-01-22 03:27:56,454 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:25:58
+2026-01-22 04:27:08,412 - easytorch-training - INFO - Iteration 50000 / 200018
+2026-01-22 04:27:08,903 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.71 (s), train/lr: 9.07e-04, train/loss: 2.5229, train/grad_norm: 2.5580, train/amp_scale: 1.0000]
+2026-01-22 04:27:08,903 - easytorch-training - INFO - Start validation.
+2026-01-22 04:27:15,629 - easytorch-training - INFO - Result <val>: [val/time: 6.55 (s), val/loss: 3.1524]
+2026-01-22 04:27:15,758 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-22 04:27:15,877 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_050000.pt saved
+2026-01-22 04:27:15,879 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:23:33
+2026-01-22 05:25:44,680 - easytorch-training - INFO - Iteration 55000 / 200018
+2026-01-22 05:25:45,168 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.70 (s), train/lr: 8.81e-04, train/loss: 2.5207, train/grad_norm: 2.4932, train/amp_scale: 1.0000]
+2026-01-22 05:25:45,169 - easytorch-training - INFO - Start validation.
+2026-01-22 05:25:51,875 - easytorch-training - INFO - Result <val>: [val/time: 6.53 (s), val/loss: 3.1542]
+2026-01-22 05:25:52,018 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_055000.pt saved
+2026-01-22 05:25:52,019 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:18:56
+2026-01-22 06:25:48,720 - easytorch-training - INFO - Iteration 60000 / 200018
+2026-01-22 06:25:49,204 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.72 (s), train/lr: 8.53e-04, train/loss: 2.5222, train/grad_norm: 2.4614, train/amp_scale: 1.0000]
+2026-01-22 06:25:49,204 - easytorch-training - INFO - Start validation.
+2026-01-22 06:25:55,879 - easytorch-training - INFO - Result <val>: [val/time: 6.50 (s), val/loss: 3.1517]
+2026-01-22 06:25:55,997 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_best_val_loss.pt saved
+2026-01-22 06:25:56,106 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_060000.pt saved
+2026-01-22 06:25:56,107 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:19:59
+2026-01-22 07:26:46,431 - easytorch-training - INFO - Iteration 65000 / 200018
+2026-01-22 07:26:46,915 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.73 (s), train/lr: 8.23e-04, train/loss: 2.5805, train/grad_norm: 3.4873, train/amp_scale: 1.0000]
+2026-01-22 07:26:46,916 - easytorch-training - INFO - Start validation.
+2026-01-22 07:26:53,580 - easytorch-training - INFO - Result <val>: [val/time: 6.49 (s), val/loss: 3.4660]
+2026-01-22 07:26:53,693 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200018/acf6c6763f53a27cd3a36171d821d82b/TimeMoE4_065000.pt saved
+2026-01-22 07:26:53,693 - easytorch-training - INFO - The estimated training finish time is 2026-01-23 10:23:37
+2026-01-22 07:37:10,038 - easytorch-training - ERROR - Traceback (most recent call last):
+  File "/home/nvidia/miniconda3/envs/zxx/lib/python3.11/site-packages/easytorch/launcher/launcher.py", line 31, in training_func
+    runner.train(cfg)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_iteration_runner.py", line 200, in train
+    self.train_iters(iteration=iteration, dataloader=self.train_data_loader)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 281, in train_iters
+    self.backward(loss, accumulating=accumulating)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 337, in backward
+    grad_norm = sum(
+                ^^^^
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 338, in <genexpr>
+    param.grad.data.norm(2).item() ** 2 for param in self.model.parameters() if param.grad is not None
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+