| | |
| | seed_everything: 123 |
| | trainer: |
| | accelerator: gpu |
| | strategy: |
| | class_path: lightning.pytorch.strategies.DeepSpeedStrategy |
| | init_args: |
| | accelerator: null |
| | zero_optimization: true |
| | stage: 2 |
| | remote_device: null |
| | offload_optimizer: false |
| | offload_parameters: true |
| | offload_params_device: cpu |
| | nvme_path: /local_nvme |
| | params_buffer_count: 5 |
| | params_buffer_size: 100000000 |
| | max_in_cpu: 1000000000 |
| | offload_optimizer_device: cpu |
| | optimizer_buffer_count: 4 |
| | block_size: 1048576 |
| | queue_depth: 8 |
| | single_submit: false |
| | overlap_events: true |
| | thread_count: 1 |
| | pin_memory: true |
| | sub_group_size: 1000000000000 |
| | contiguous_gradients: true |
| | overlap_comm: true |
| | allgather_partitions: true |
| | reduce_scatter: true |
| | allgather_bucket_size: 200000000 |
| | reduce_bucket_size: 200000000 |
| | zero_allow_untested_optimizer: true |
| | logging_batch_size_per_gpu: auto |
| | config: null |
| | logging_level: 30 |
| | parallel_devices: null |
| | cluster_environment: null |
| | loss_scale: 0.0 |
| | initial_scale_power: 16 |
| | loss_scale_window: 1000 |
| | hysteresis: 2 |
| | min_loss_scale: 1 |
| | partition_activations: false |
| | cpu_checkpointing: false |
| | contiguous_memory_optimization: false |
| | synchronize_checkpoint_boundary: false |
| | load_full_weights: false |
| | precision_plugin: null |
| | process_group_backend: null |
| | devices: 8 |
| | num_nodes: 1 |
| | precision: bf16-true |
| | logger: |
| | class_path: lightning.pytorch.loggers.TensorBoardLogger |
| | init_args: |
| | save_dir: /media/logs |
| | name: main |
| | version: null |
| | log_graph: false |
| | default_hp_metric: true |
| | prefix: '' |
| | sub_dir: null |
| | comment: '' |
| | purge_step: null |
| | max_queue: 10 |
| | flush_secs: 120 |
| | filename_suffix: '' |
| | callbacks: null |
| | fast_dev_run: false |
| | max_epochs: 2 |
| | min_epochs: null |
| | max_steps: -1 |
| | min_steps: null |
| | max_time: null |
| | limit_train_batches: null |
| | limit_val_batches: null |
| | limit_test_batches: null |
| | limit_predict_batches: null |
| | overfit_batches: 0.0 |
| | val_check_interval: null |
| | check_val_every_n_epoch: 1 |
| | num_sanity_val_steps: 0 |
| | log_every_n_steps: 1 |
| | enable_checkpointing: null |
| | enable_progress_bar: null |
| | enable_model_summary: null |
| | accumulate_grad_batches: 8 |
| | gradient_clip_val: null |
| | gradient_clip_algorithm: null |
| | deterministic: null |
| | benchmark: null |
| | inference_mode: true |
| | use_distributed_sampler: true |
| | profiler: null |
| | detect_anomaly: false |
| | barebones: false |
| | plugins: null |
| | sync_batchnorm: false |
| | reload_dataloaders_every_n_epochs: 0 |
| | default_root_dir: null |
| | model: |
| | config: |
| | model_name: Mistral-7B-v0.2 |
| | dtype: bfloat16 |
| | num_thoughts: 2 |
| | thought_length: 8 |
| | lookahead_tokens: 4 |
| | embedding_grad_weights: 100.0 |
| | temperature: 1.0 |
| | do_sample: true |
| | train_max_length: 120 |
| | offload_cache: false |
| | top_k: null |
| | top_p: null |
| | checkpoint_dir: /media/models/unsloth/Mistral-7B-v0.2 |
| | weight_decay: 0.001 |
| | warmup_steps: 20 |
| | policy_weight: 1.0 |
| | init_lr: 1.0e-06 |
| | optimizer: |
| | class_path: torch.optim.AdamW |
| | init_args: |
| | lr: 1.0e-06 |
| | betas: |
| | - 0.9 |
| | - 0.999 |
| | eps: 1.0e-08 |
| | weight_decay: 0.001 |
| | amsgrad: false |
| | maximize: false |
| | foreach: null |
| | capturable: false |
| | differentiable: false |
| | fused: null |
| | scheduler: null |
| | ckpt_path: null |
| | data: |
| | class_path: src.dataset.OpenWebMathDataModule |
| | init_args: |
| | data_path: /media/datasets/openwebmath |
| | tokenizer: |
| | class_path: src.dataset.SpecialTokenizer |
| | init_args: |
| | checkpoint_dir: /media/models/unsloth/Mistral-7B-v0.2 |
| | batch_size: 1 |
| | max_seq_length: 120 |
| | num_samples: 2048 |
| | ignore_index: -100 |
| | val_split_fraction: 0.125 |
| | seed: 42 |
| | num_workers: 1 |
| |
|
