Create training_config.json

training_config.json

@@ -0,0 +1,143 @@
+{
+  "model_name_or_path": "meta-llama/Meta-Llama-3.1-70B",
+  "output_dir": "./checkpoints/helion-2.5-rnd",
+  "overwrite_output_dir": true,
+  "do_train": true,
+  "do_eval": true,
+  "evaluation_strategy": "steps",
+  "eval_steps": 500,
+  "per_device_train_batch_size": 4,
+  "per_device_eval_batch_size": 4,
+  "gradient_accumulation_steps": 8,
+  "learning_rate": 2e-05,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "adam_epsilon": 1e-08,
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 3,
+  "max_steps": 150000,
+  "lr_scheduler_type": "cosine_with_restarts",
+  "warmup_steps": 2000,
+  "logging_dir": "./logs",
+  "logging_strategy": "steps",
+  "logging_steps": 10,
+  "save_strategy": "steps",
+  "save_steps": 1000,
+  "save_total_limit": 5,
+  "fp16": false,
+  "bf16": true,
+  "dataloader_num_workers": 8,
+  "dataloader_pin_memory": true,
+  "gradient_checkpointing": true,
+  "gradient_checkpointing_kwargs": {
+    "use_reentrant": false
+  },
+  "deepspeed": {
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+      "stage": 2,
+      "offload_optimizer": {
+        "device": "cpu",
+        "pin_memory": true
+      },
+      "offload_param": {
+        "device": "cpu",
+        "pin_memory": true
+      },
+      "overlap_comm": true,
+      "contiguous_gradients": true,
+      "reduce_bucket_size": 5e7,
+      "stage3_prefetch_bucket_size": 5e7,
+      "stage3_param_persistence_threshold": 1e5
+    },
+    "fp16": {
+      "enabled": false
+    },
+    "bf16": {
+      "enabled": true
+    },
+    "optimizer": {
+      "type": "AdamW",
+      "params": {
+        "lr": "auto",
+        "betas": "auto",
+        "eps": "auto",
+        "weight_decay": "auto"
+      }
+    },
+    "scheduler": {
+      "type": "WarmupDecayLR",
+      "params": {
+        "warmup_min_lr": "auto",
+        "warmup_max_lr": "auto",
+        "warmup_num_steps": "auto",
+        "total_num_steps": "auto"
+      }
+    },
+    "zero_allow_untested_optimizer": true,
+    "wall_clock_breakdown": false
+  },
+  "fsdp": "",
+  "fsdp_config": {},
+  "report_to": ["tensorboard", "wandb"],
+  "run_name": "helion-2.5-rnd",
+  "disable_tqdm": false,
+  "remove_unused_columns": false,
+  "label_names": ["labels"],
+  "load_best_model_at_end": true,
+  "metric_for_best_model": "eval_loss",
+  "greater_is_better": false,
+  "ignore_data_skip": false,
+  "ddp_timeout": 1800,
+  "torch_compile": false,
+  "torch_compile_backend": "inductor",
+  "torch_compile_mode": null,
+  "optim": "adamw_torch_fused",
+  "group_by_length": false,
+  "length_column_name": "length",
+  "ddp_find_unused_parameters": false,
+  "ddp_bucket_cap_mb": null,
+  "ddp_broadcast_buffers": null,
+  "dataset_config": {
+    "scientific_papers": {
+      "path": "arxiv_papers",
+      "split": "train",
+      "weight": 0.25
+    },
+    "code_repositories": {
+      "path": "the-stack-dedup",
+      "split": "train",
+      "weight": 0.25
+    },
+    "mathematical_proofs": {
+      "path": "math_qa",
+      "split": "train",
+      "weight": 0.15
+    },
+    "conversational_data": {
+      "path": "sharegpt",
+      "split": "train",
+      "weight": 0.20
+    },
+    "multilingual_corpus": {
+      "path": "mc4",
+      "split": "train",
+      "weight": 0.15
+    }
+  },
+  "data_preprocessing": {
+    "max_seq_length": 131072,
+    "truncation": true,
+    "padding": "max_length",
+    "return_tensors": "pt"
+  },
+  "model_config_updates": {
+    "use_cache": false,
+    "attention_dropout": 0.0,
+    "hidden_dropout": 0.0
+  }
+}