Upload Main_200014

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_005000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ed177201c2e3c434e8e2fa72de8652904410576e53ec295ae68d9207bde09b
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_010000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1511b7cf6e742d9a9746df22aa738efcf3641245a45ec6b0bcbdb18e0e892cee
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_015000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79094b4ed8e8a15eafdfab9310dc2179d050c951cfcf8c039418879536dd02c9
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_020000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:934f3de40b880d2a5c3f39e4653c1f19c66621de722b9937c4c7ea75a62814e1
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_025000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3b1571ca796987c8ddd72371478a2c7263c50f935e3c41adee8a7ae66887db8
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_030000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d997dad6cbd7e56b5a3b856e76d71ade438c342d4701586bbef5254993dfb70c
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_035000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa4e413639241176033a719569c4efaa7a4fd8f4c00b0c96fa4acf2c8ee806db
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_040000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c40c4a3cdcfe53fb2dfb754a5499fae7b58281fae491d59c0460e4280e0c637
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_045000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17bc2f9f1d5550a48a45bac7ec6a3eb3c91308671ebbb4205029ae7dbb72ddae
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_050000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6b83b6261b8327b56d9d692ea535ac90d0e8f988a68a8c0c44f7ffce1108ff1
+size 151685888

Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70f12059fa4c52931b265aae8b72a1aa1b93e17c3c16dcbddd357fc2ac88c11d
+size 151688871

Main_200014/50a47da570c7010fe8e0a1c34a78d536/cfg.txt

@@ -0,0 +1,55 @@
+DESCRIPTION: TimeMoE Base
+DEVICE: gpu
+DEVICE_NUM: 3
+RUNNER: <class 'baselines.TimeMoE4.runner.runner.TimeMoERunner'>
+MODEL:
+  NAME: TimeMoE4
+  ARCH: <class 'baselines.TimeMoE4.arch.timemoe.TimeMoE4'>
+  PARAM:
+    model_id: baselines/TimeMoE/ckpt/TimeMoE-50M
+    from_pretrained: False
+    context_length: 4079
+    trust_remote_code: True
+  DTYPE: bfloat16
+METRICS:
+  FUNCS:
+TRAIN:
+  COMPILE_MODEL: True
+  NUM_ITERATIONS: 200014
+  CKPT_SAVE_DIR: checkpoints/TimeMoE4/Main_200014
+  CKPT_SAVE_STRATEGY: 5000
+  LOSS: fake_loss
+  OPTIM:
+    TYPE: AdamW
+    PARAM:
+      lr: 0.001
+      betas: (0.9, 0.95)
+      fused: True
+  LR_SCHEDULER:
+    TYPE: CosineWarmup
+    PARAM:
+      num_warmup_steps: 10000
+      num_training_steps: 200014
+  CLIP_GRAD_PARAM:
+    max_norm: 1.0
+  DATA:
+    BATCH_SIZE: 85
+    SHUFFLE: True
+    PIN_MEMORY: True
+    PREFETCH: True
+  GRAD_ACCUMULATION_STEPS: 1
+VAL:
+  INTERVAL: 5000
+  DATA:
+    BATCH_SIZE: 170
+EVAL:
+  USE_GPU: True
+DATASET:
+  NAME: Main
+  TYPE: <class 'baselines.TimeMoE4.data.mix_dataset_v2.MixedSourceDataset_v2'>
+  PARAM:
+    num_valid_samples: 1000
+INFERENCE:
+  GENERATION_PARAMS:
+    normalize: True
+MD5: 50a47da570c7010fe8e0a1c34a78d536

Main_200014/50a47da570c7010fe8e0a1c34a78d536/tensorboard/events.out.tfevents.1768807194.brev-5x9knwe1p.3708293.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40697a2fea2646111cad3d0310712e4422d6e6d9f69d51c3c65912b2a7002091
+size 11234316

Main_200014/50a47da570c7010fe8e0a1c34a78d536/timemoe_base.py

@@ -0,0 +1,126 @@
+# 采样概率变化
+
+import os
+import sys
+from easydict import EasyDict
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ..arch import TimeMoE4
+from ..data import MixedSourceDataset_v2
+from ..runner import TimeMoERunner
+from ..loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+pretrained = False  # Whether to use a pretrained model
+
+MODEL_ARCH = TimeMoE4
+
+MODEL_PARAM = {
+    'model_id': "baselines/TimeMoE/ckpt/TimeMoE-50M",
+    'from_pretrained': pretrained,
+    'context_length': 4079,
+    'trust_remote_code': True,
+}
+DATA_NAME = "Main"
+
+# N = 20_000_000
+# batch size = 16*8
+# 20_000_000 / 16 / 8 = 156250 iterations
+# 20_000_000 * 4096 / 16 / 8 / 4096 = 156_250
+
+NUM_ITERATIONS = 200_014 # 总轮数   20_000_000 * 4096 / 16 / 4 / 4096 = 312,500
+VAL_ITERATION_INTERVAL = 5_000 # 每VAL_ITERATION_INTERVAL执行一次验证
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'TimeMoE Base'
+CFG.DEVICE = 'gpu'
+CFG.DEVICE_NUM = 3
+# Runner
+CFG.RUNNER = TimeMoERunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+# CFG.MODEL.DTYPE= 'float32'
+
+############################## Metrics Configuration ##############################
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({})
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = True
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+CFG.TRAIN.CKPT_SAVE_STRATEGY = VAL_ITERATION_INTERVAL * 1 # 保存策略，每VAL_ITERATION_INTERVAL * 5保存一次模型
+CFG.TRAIN.LOSS = fake_loss
+# Optimizer settings
+CFG.TRAIN.OPTIM = EasyDict()
+CFG.TRAIN.OPTIM.TYPE = "AdamW"
+CFG.TRAIN.OPTIM.PARAM = {
+    "lr": 1e-3,
+    "betas": (0.9, 0.95),
+    # "betas": (0.9, 0.98),
+    "fused": True,
+    # "weight_decay": 1e-1,
+}
+# Learning rate scheduler settings
+CFG.TRAIN.LR_SCHEDULER = EasyDict()
+CFG.TRAIN.LR_SCHEDULER.TYPE = "CosineWarmup"
+CFG.TRAIN.LR_SCHEDULER.PARAM = {
+    'num_warmup_steps': 10_000, # 10k
+    'num_training_steps': NUM_ITERATIONS,
+}
+CFG.TRAIN.CLIP_GRAD_PARAM = {
+    'max_norm': 1.0
+}
+# Train data loader settings
+CFG.TRAIN.DATA = EasyDict()
+CFG.TRAIN.DATA.BATCH_SIZE = 85  # 16  /  4
+CFG.TRAIN.DATA.SHUFFLE = True # has to be False
+CFG.TRAIN.DATA.PIN_MEMORY = True
+CFG.TRAIN.DATA.PREFETCH = True
+CFG.TRAIN.GRAD_ACCUMULATION_STEPS = 1
+# CFG.TRAIN.DATA.NUM_WORKERS = 4
+
+############################## Validation Configuration ##############################
+CFG.VAL = EasyDict()
+CFG.VAL.INTERVAL = VAL_ITERATION_INTERVAL
+CFG.VAL.DATA = EasyDict()
+CFG.VAL.DATA.BATCH_SIZE = 170  # 32  /  8
+
+############################## Evaluation Configuration ##############################
+
+CFG.EVAL = EasyDict()
+# Evaluation parameters
+CFG.EVAL.USE_GPU = True # Whether to use GPU for evaluation. Default: True
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = MixedSourceDataset_v2
+CFG.DATASET.PARAM = EasyDict({
+    'num_valid_samples': 1000
+})
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+    'normalize': not pretrained
+})
+

Main_200014/50a47da570c7010fe8e0a1c34a78d536/training_log_20260119071944.log

@@ -0,0 +1,114 @@
+2026-01-19 07:19:44,428 - easytorch-training - INFO - Initializing training.
+2026-01-19 07:19:44,429 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-19 07:19:44,429 - easytorch-training - INFO - Building training data loader.
+2026-01-19 07:19:54,179 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-19 07:19:54,179 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-19 07:19:54,179 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-19 07:19:54,180 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-19 07:19:54,182 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.001
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-19 07:19:54,182 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x753064a97dd0>
+2026-01-19 07:19:54,184 - easytorch-training - INFO - Initializing validation.
+2026-01-19 07:19:54,184 - easytorch-training - INFO - Building val data loader.
+2026-01-19 07:19:54,463 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-19 07:20:20,862 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-19 07:20:20,862 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-19 07:20:20,862 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-19 07:20:20,863 - easytorch-training - INFO - Number of parameters: 12628992
+2026-01-19 07:20:20,863 - easytorch-training - INFO - Training with 3 GPUs, batch size per GPUs: 85, grad_accumulation_steps: 1
+2026-01-19 07:20:20,863 - easytorch-training - INFO - Effective batch size: 255
+2026-01-19 08:26:13,305 - easytorch-training - INFO - Iteration 5000 / 200014
+2026-01-19 08:26:13,725 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.79 (s), train/lr: 2.50e-04, train/loss: 3.9592, train/grad_norm: 6.1824, train/amp_scale: 1.0000]
+2026-01-19 08:26:13,725 - easytorch-training - INFO - Start validation.
+2026-01-19 08:26:27,598 - easytorch-training - INFO - Result <val>: [val/time: 13.70 (s), val/loss: 3.3753]
+2026-01-19 08:26:27,704 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 08:26:27,815 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_005000.pt saved
+2026-01-19 08:26:27,816 - easytorch-training - INFO - The estimated training finish time is 2026-01-21 03:25:10
+2026-01-19 09:20:49,426 - easytorch-training - INFO - Iteration 10000 / 200014
+2026-01-19 09:20:49,846 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.65 (s), train/lr: 7.50e-04, train/loss: 3.2398, train/grad_norm: 2.2217, train/amp_scale: 1.0000]
+2026-01-19 09:20:49,846 - easytorch-training - INFO - Start validation.
+2026-01-19 09:20:56,465 - easytorch-training - INFO - Result <val>: [val/time: 6.44 (s), val/loss: 3.3116]
+2026-01-19 09:20:56,596 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 09:20:56,713 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_010000.pt saved
+2026-01-19 09:20:56,714 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 23:32:28
+2026-01-19 10:15:17,743 - easytorch-training - INFO - Iteration 15000 / 200014
+2026-01-19 10:15:18,161 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.65 (s), train/lr: 9.99e-04, train/loss: 3.1413, train/grad_norm: 1.9954, train/amp_scale: 1.0000]
+2026-01-19 10:15:18,161 - easytorch-training - INFO - Start validation.
+2026-01-19 10:15:24,759 - easytorch-training - INFO - Result <val>: [val/time: 6.42 (s), val/loss: 3.2741]
+2026-01-19 10:15:24,887 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 10:15:25,004 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_015000.pt saved
+2026-01-19 10:15:25,005 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 22:14:45
+2026-01-19 11:08:50,142 - easytorch-training - INFO - Iteration 20000 / 200014
+2026-01-19 11:08:50,559 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.64 (s), train/lr: 9.96e-04, train/loss: 3.0921, train/grad_norm: 2.0023, train/amp_scale: 1.0000]
+2026-01-19 11:08:50,559 - easytorch-training - INFO - Start validation.
+2026-01-19 11:08:57,149 - easytorch-training - INFO - Result <val>: [val/time: 6.42 (s), val/loss: 3.2675]
+2026-01-19 11:08:57,264 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 11:08:57,374 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_020000.pt saved
+2026-01-19 11:08:57,375 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 21:26:35
+2026-01-19 12:01:38,196 - easytorch-training - INFO - Iteration 25000 / 200014
+2026-01-19 12:01:38,617 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.63 (s), train/lr: 9.89e-04, train/loss: 3.0706, train/grad_norm: 1.9924, train/amp_scale: 1.0000]
+2026-01-19 12:01:38,617 - easytorch-training - INFO - Start validation.
+2026-01-19 12:01:45,244 - easytorch-training - INFO - Result <val>: [val/time: 6.45 (s), val/loss: 3.2473]
+2026-01-19 12:01:45,377 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 12:01:45,497 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_025000.pt saved
+2026-01-19 12:01:45,497 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 20:51:47
+2026-01-19 12:55:14,438 - easytorch-training - INFO - Iteration 30000 / 200014
+2026-01-19 12:55:14,860 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.64 (s), train/lr: 9.79e-04, train/loss: 3.0554, train/grad_norm: 2.0358, train/amp_scale: 1.0000]
+2026-01-19 12:55:14,861 - easytorch-training - INFO - Start validation.
+2026-01-19 12:55:21,487 - easytorch-training - INFO - Result <val>: [val/time: 6.45 (s), val/loss: 3.2401]
+2026-01-19 12:55:21,614 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 12:55:21,732 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_030000.pt saved
+2026-01-19 12:55:21,733 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 20:33:56
+2026-01-19 13:49:00,480 - easytorch-training - INFO - Iteration 35000 / 200014
+2026-01-19 13:49:00,899 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.64 (s), train/lr: 9.66e-04, train/loss: 3.0499, train/grad_norm: 2.0126, train/amp_scale: 1.0000]
+2026-01-19 13:49:00,900 - easytorch-training - INFO - Start validation.
+2026-01-19 13:49:07,719 - easytorch-training - INFO - Result <val>: [val/time: 6.64 (s), val/loss: 3.2354]
+2026-01-19 13:49:07,840 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 13:49:07,951 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_035000.pt saved
+2026-01-19 13:49:07,952 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 20:22:07
+2026-01-19 14:43:01,179 - easytorch-training - INFO - Iteration 40000 / 200014
+2026-01-19 14:43:01,597 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.65 (s), train/lr: 9.49e-04, train/loss: 3.0318, train/grad_norm: 2.0432, train/amp_scale: 1.0000]
+2026-01-19 14:43:01,597 - easytorch-training - INFO - Start validation.
+2026-01-19 14:43:08,189 - easytorch-training - INFO - Result <val>: [val/time: 6.42 (s), val/loss: 3.2270]
+2026-01-19 14:43:08,316 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 14:43:08,434 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_040000.pt saved
+2026-01-19 14:43:08,434 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 20:14:28
+2026-01-19 15:36:00,962 - easytorch-training - INFO - Iteration 45000 / 200014
+2026-01-19 15:36:01,379 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.63 (s), train/lr: 9.29e-04, train/loss: 3.0320, train/grad_norm: 2.0299, train/amp_scale: 1.0000]
+2026-01-19 15:36:01,380 - easytorch-training - INFO - Start validation.
+2026-01-19 15:36:07,990 - easytorch-training - INFO - Result <val>: [val/time: 6.44 (s), val/loss: 3.2236]
+2026-01-19 15:36:08,124 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_best_val_loss.pt saved
+2026-01-19 15:36:08,245 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_045000.pt saved
+2026-01-19 15:36:08,246 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 20:04:00
+2026-01-19 16:29:58,457 - easytorch-training - INFO - Iteration 50000 / 200014
+2026-01-19 16:29:58,876 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.64 (s), train/lr: 9.07e-04, train/loss: 3.0149, train/grad_norm: 2.0357, train/amp_scale: 1.0000]
+2026-01-19 16:29:58,876 - easytorch-training - INFO - Start validation.
+2026-01-19 16:30:05,501 - easytorch-training - INFO - Result <val>: [val/time: 6.45 (s), val/loss: 3.2239]
+2026-01-19 16:30:05,621 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200014/50a47da570c7010fe8e0a1c34a78d536/TimeMoE4_050000.pt saved
+2026-01-19 16:30:05,622 - easytorch-training - INFO - The estimated training finish time is 2026-01-20 19:59:29
+2026-01-19 17:17:11,237 - easytorch-training - ERROR - Traceback (most recent call last):
+  File "/home/nvidia/miniconda3/envs/zxx/lib/python3.11/site-packages/easytorch/launcher/launcher.py", line 31, in training_func
+    runner.train(cfg)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_iteration_runner.py", line 200, in train
+    self.train_iters(iteration=iteration, dataloader=self.train_data_loader)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 281, in train_iters
+    self.backward(loss, accumulating=accumulating)
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 337, in backward
+    grad_norm = sum(
+                ^^^^
+  File "/lp-dev/zhouxx/BasicTS/basicts/runners/base_utsf_runner.py", line 338, in <genexpr>
+    param.grad.data.norm(2).item() ** 2 for param in self.model.parameters() if param.grad is not None
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+KeyboardInterrupt
+