Upload configs/telecom-1.35M-v2.yaml with huggingface_hub

configs/telecom-1.35M-v2.yaml

@@ -0,0 +1,50 @@
+# =============================================================================
+# Telecom 1.35M-v2 Fine-tuning Config
+# Date: 2025-01-15
+# Dataset: ~1.35M telecom examples (augmented network slicing & NF config)
+# GPUs: 4x H100 NVL (devices 0-3)
+# Strategy: Conservative 1 epoch to avoid catastrophic forgetting
+# Base: telecom-1.27M settings, adjusted iterations for larger dataset
+# =============================================================================
+
+train:
+  train_iters: 10500
+  global_batch_size: 128    # 4 GPUs
+  micro_batch_size: 4
+
+model:
+  expert_model_parallel_size: 4  # MUST match GPU count
+  moe_token_dispatcher_type: alltoall
+
+optimizer:
+  lr: 5e-5  # Conservative to prevent forgetting
+
+scheduler:
+  lr_warmup_iters: 525      # 5% of train_iters
+  lr_decay_iters: 10500     # Match train_iters
+
+logger:
+  log_interval: 10
+
+checkpoint:
+  save_interval: 1000
+  pretrained_checkpoint: /models/nemotron-30b-megatron
+  save: /models/telecom-1.35M-v2-lora
+  # load: /models/telecom-1.35M-v2-lora  # Uncomment to resume
+
+dataset:
+  dataset_name: json
+  dataset_root: /models/telecom-1.35M-v2
+  hf_kwargs:
+    data_files:
+      train: /models/telecom-1.35M-v2/train.jsonl
+      validation: /models/telecom-1.35M-v2/validation.jsonl
+      test: /models/telecom-1.35M-v2/test.jsonl
+  dataloader_type: batch
+  data_sharding: true
+  do_test: false
+  do_validation: true
+  num_workers: 4
+  pin_memory: true
+  seed: 42
+  seq_length: 2048