Upload Main_200024

Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_005000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50982da1d3b1ca0ab78ace98ed0125bb192b4e9a824b8ecf28a898b8fb99c147
+size 151982040

Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_010000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a7f0fd27c4a91675781c7607e729b0b863460b67d3836972bfcc8e4d9b06425
+size 151982040

Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_best_val_loss.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c08065e8ff25d8945e65fe0680c2b9289bbe5cd60a08653df7696f38d8c3634a
+size 151985051

Main_200024/ac18139a247064d67b6c0f94e201f2ac/cfg.txt

@@ -0,0 +1,55 @@
+DESCRIPTION: TimeMoE Base
+DEVICE: gpu
+DEVICE_NUM: 3
+RUNNER: <class 'baselines.TimeMoE4.runner.runner.TimeMoERunner'>
+MODEL:
+  NAME: TimeMoE4
+  ARCH: <class 'baselines.TimeMoE4.arch.timemoe.TimeMoE4'>
+  PARAM:
+    model_id: baselines/TimeMoE/ckpt/TimeMoE-50M
+    from_pretrained: False
+    context_length: 4079
+    trust_remote_code: True
+  DTYPE: bfloat16
+METRICS:
+  FUNCS:
+TRAIN:
+  COMPILE_MODEL: True
+  NUM_ITERATIONS: 200024
+  CKPT_SAVE_DIR: checkpoints/TimeMoE4/Main_200024
+  CKPT_SAVE_STRATEGY: 5000
+  LOSS: fake_loss
+  OPTIM:
+    TYPE: AdamW
+    PARAM:
+      lr: 0.001
+      betas: (0.9, 0.95)
+      fused: True
+  LR_SCHEDULER:
+    TYPE: CosineWarmup
+    PARAM:
+      num_warmup_steps: 10000
+      num_training_steps: 200024
+  CLIP_GRAD_PARAM:
+    max_norm: 1.0
+  DATA:
+    BATCH_SIZE: 85
+    SHUFFLE: True
+    PIN_MEMORY: True
+    PREFETCH: True
+  GRAD_ACCUMULATION_STEPS: 1
+VAL:
+  INTERVAL: 5000
+  DATA:
+    BATCH_SIZE: 170
+EVAL:
+  USE_GPU: True
+DATASET:
+  NAME: Main
+  TYPE: <class 'baselines.TimeMoE4.data.mix_dataset_v2.MixedSourceDataset_v2'>
+  PARAM:
+    num_valid_samples: 1000
+INFERENCE:
+  GENERATION_PARAMS:
+    normalize: True
+MD5: ac18139a247064d67b6c0f94e201f2ac

Main_200024/ac18139a247064d67b6c0f94e201f2ac/tensorboard/events.out.tfevents.1769416276.brev-5x9knwe1p.1731692.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72c5bde2a05d31d638a9c7cc5d3535177629410a5c938959866bddcdc182ecb9
+size 2626580

Main_200024/ac18139a247064d67b6c0f94e201f2ac/timemoe_base.py

@@ -0,0 +1,126 @@
+# 采样概率变化
+
+import os
+import sys
+from easydict import EasyDict
+sys.path.append(os.path.abspath(__file__ + '/../../..'))
+
+from ..arch import TimeMoE4
+from ..data import MixedSourceDataset_v2
+from ..runner import TimeMoERunner
+from ..loss import fake_loss
+
+
+############################## Hot Parameters ##############################
+# Dataset & Metrics configuration
+# Model architecture and parameters
+
+pretrained = False  # Whether to use a pretrained model
+
+MODEL_ARCH = TimeMoE4
+
+MODEL_PARAM = {
+    'model_id': "baselines/TimeMoE/ckpt/TimeMoE-50M",
+    'from_pretrained': pretrained,
+    'context_length': 4079,
+    'trust_remote_code': True,
+}
+DATA_NAME = "Main"
+
+# N = 20_000_000
+# batch size = 16*8
+# 20_000_000 / 16 / 8 = 156250 iterations
+# 20_000_000 * 4096 / 16 / 8 / 4096 = 156_250
+
+NUM_ITERATIONS = 200_024 # 总轮数   20_000_000 * 4096 / 16 / 4 / 4096 = 312,500
+VAL_ITERATION_INTERVAL = 5_000 # 每VAL_ITERATION_INTERVAL执行一次验证
+
+############################## General Configuration ##############################
+CFG = EasyDict()
+# General settings
+CFG.DESCRIPTION = 'TimeMoE Base'
+CFG.DEVICE = 'gpu'
+CFG.DEVICE_NUM = 3
+# Runner
+CFG.RUNNER = TimeMoERunner
+
+############################## Model Configuration ################################
+CFG.MODEL = EasyDict()
+CFG.MODEL.NAME = MODEL_ARCH.__name__
+CFG.MODEL.ARCH = MODEL_ARCH
+CFG.MODEL.PARAM = MODEL_PARAM
+CFG.MODEL.DTYPE= 'bfloat16'
+# CFG.MODEL.DTYPE= 'float32'
+
+############################## Metrics Configuration ##############################
+CFG.METRICS = EasyDict()
+# Metrics settings
+CFG.METRICS.FUNCS = EasyDict({})
+
+############################## Training Configuration ##############################
+CFG.TRAIN = EasyDict()
+CFG.TRAIN.COMPILE_MODEL = True
+CFG.TRAIN.NUM_ITERATIONS = NUM_ITERATIONS
+CFG.TRAIN.CKPT_SAVE_DIR = os.path.join(
+    'checkpoints',
+    MODEL_ARCH.__name__,
+    '_'.join([DATA_NAME, str(CFG.TRAIN.NUM_ITERATIONS)])
+)
+CFG.TRAIN.CKPT_SAVE_STRATEGY = VAL_ITERATION_INTERVAL * 1 # 保存策略，每VAL_ITERATION_INTERVAL * 5保存一次模型
+CFG.TRAIN.LOSS = fake_loss
+# Optimizer settings
+CFG.TRAIN.OPTIM = EasyDict()
+CFG.TRAIN.OPTIM.TYPE = "AdamW"
+CFG.TRAIN.OPTIM.PARAM = {
+    "lr": 1e-3,
+    "betas": (0.9, 0.95),
+    # "betas": (0.9, 0.98),
+    "fused": True,
+    # "weight_decay": 1e-1,
+}
+# Learning rate scheduler settings
+CFG.TRAIN.LR_SCHEDULER = EasyDict()
+CFG.TRAIN.LR_SCHEDULER.TYPE = "CosineWarmup"
+CFG.TRAIN.LR_SCHEDULER.PARAM = {
+    'num_warmup_steps': 10_000, # 10k
+    'num_training_steps': NUM_ITERATIONS,
+}
+CFG.TRAIN.CLIP_GRAD_PARAM = {
+    'max_norm': 1.0
+}
+# Train data loader settings
+CFG.TRAIN.DATA = EasyDict()
+CFG.TRAIN.DATA.BATCH_SIZE = 85  # 16  /  4
+CFG.TRAIN.DATA.SHUFFLE = True # has to be False
+CFG.TRAIN.DATA.PIN_MEMORY = True
+CFG.TRAIN.DATA.PREFETCH = True
+CFG.TRAIN.GRAD_ACCUMULATION_STEPS = 1
+# CFG.TRAIN.DATA.NUM_WORKERS = 4
+
+############################## Validation Configuration ##############################
+CFG.VAL = EasyDict()
+CFG.VAL.INTERVAL = VAL_ITERATION_INTERVAL
+CFG.VAL.DATA = EasyDict()
+CFG.VAL.DATA.BATCH_SIZE = 170  # 32  /  8
+
+############################## Evaluation Configuration ##############################
+
+CFG.EVAL = EasyDict()
+# Evaluation parameters
+CFG.EVAL.USE_GPU = True # Whether to use GPU for evaluation. Default: True
+
+############################## Dataset Configuration ##############################
+CFG.DATASET = EasyDict()
+# Dataset settings
+CFG.DATASET.NAME = DATA_NAME
+CFG.DATASET.TYPE = MixedSourceDataset_v2
+CFG.DATASET.PARAM = EasyDict({
+    'num_valid_samples': 1000
+})
+
+############################## Inference Configuration ##############################
+CFG.INFERENCE = EasyDict()
+CFG.INFERENCE.GENERATION_PARAMS = EasyDict({
+    'normalize': not pretrained
+})
+

Main_200024/ac18139a247064d67b6c0f94e201f2ac/training_log_20260126083104.log

@@ -0,0 +1,44 @@
+2026-01-26 08:31:04,758 - easytorch-training - INFO - Initializing training.
+2026-01-26 08:31:04,758 - easytorch-training - INFO - Set clip grad, param: {'max_norm': 1.0}
+2026-01-26 08:31:04,759 - easytorch-training - INFO - Building training data loader.
+2026-01-26 08:31:16,361 - easytorch-training - INFO - MixedSourceDataset initialized for 'train' mode.
+2026-01-26 08:31:16,362 - easytorch-training - INFO -   - real: 3201174 samples
+2026-01-26 08:31:16,362 - easytorch-training - INFO -   - synth: 2000000 samples
+2026-01-26 08:31:16,362 - easytorch-training - INFO - Train dataset length: 3201174
+2026-01-26 08:31:16,364 - easytorch-training - INFO - Set optim: AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: (0.9, 0.95)
+    capturable: False
+    differentiable: False
+    eps: 1e-08
+    foreach: None
+    fused: True
+    lr: 0.001
+    maximize: False
+    weight_decay: 0.01
+)
+2026-01-26 08:31:16,365 - easytorch-training - INFO - Set lr_scheduler: <basicts.runners.optim.lr_schedulers.CosineWarmup object at 0x73f84b49a210>
+2026-01-26 08:31:16,366 - easytorch-training - INFO - Initializing validation.
+2026-01-26 08:31:16,367 - easytorch-training - INFO - Building val data loader.
+2026-01-26 08:31:16,665 - easytorch-training - INFO - Worker 0 initialized for cauker_univariate.
+2026-01-26 08:31:42,629 - easytorch-training - INFO - MixedSourceDataset initialized for 'valid' mode.
+2026-01-26 08:31:42,629 - easytorch-training - INFO -   - real: 1000 samples
+2026-01-26 08:31:42,629 - easytorch-training - INFO - Valid dataset length: 1000
+2026-01-26 08:31:42,630 - easytorch-training - INFO - Number of parameters: 12653568
+2026-01-26 08:31:42,631 - easytorch-training - INFO - Training with 3 GPUs, batch size per GPUs: 85, grad_accumulation_steps: 1
+2026-01-26 08:31:42,631 - easytorch-training - INFO - Effective batch size: 255
+2026-01-26 09:27:50,579 - easytorch-training - INFO - Iteration 5000 / 200024
+2026-01-26 09:27:51,075 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.67 (s), train/lr: 2.50e-04, train/loss: 3.4470, train/grad_norm: 7.4594, train/amp_scale: 1.0000]
+2026-01-26 09:27:51,076 - easytorch-training - INFO - Start validation.
+2026-01-26 09:28:03,753 - easytorch-training - INFO - Result <val>: [val/time: 12.50 (s), val/loss: 3.3114]
+2026-01-26 09:28:03,874 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_best_val_loss.pt saved
+2026-01-26 09:28:03,994 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_005000.pt saved
+2026-01-26 09:28:03,995 - easytorch-training - INFO - The estimated training finish time is 2026-01-27 22:06:13
+2026-01-26 10:24:35,419 - easytorch-training - INFO - Iteration 10000 / 200024
+2026-01-26 10:24:35,917 - easytorch-training - INFO - Result <train>: [train/iter_time: 0.68 (s), train/lr: 7.50e-04, train/loss: 2.6932, train/grad_norm: 3.3237, train/amp_scale: 1.0000]
+2026-01-26 10:24:35,917 - easytorch-training - INFO - Start validation.
+2026-01-26 10:24:42,648 - easytorch-training - INFO - Result <val>: [val/time: 6.55 (s), val/loss: 3.2474]
+2026-01-26 10:24:42,781 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_best_val_loss.pt saved
+2026-01-26 10:24:42,900 - easytorch-training - INFO - Checkpoint checkpoints/TimeMoE4/Main_200024/ac18139a247064d67b6c0f94e201f2ac/TimeMoE4_010000.pt saved
+2026-01-26 10:24:42,902 - easytorch-training - INFO - The estimated training finish time is 2026-01-27 22:12:04