Upload run_fsdp.text with huggingface_hub

run_fsdp.text

@@ -0,0 +1,58 @@
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True accelerate launch --config_file "configs/fsdp_config.yaml" \
+--main_process_ip "172.17.0.2" \
+--main_process_port "29080" \
+--machine_rank $BACKENDAI_CLUSTER_LOCAL_RANK \
+--num_processes 4 \
+--num_machines 1 \
+--monitor_interval 10 \
+--num_cpu_threads_per_process 8 \
+train.py \
+--seed 100 \
+--model_name_or_path "meta-llama/Meta-Llama-3.1-8B-Instruct" \
+--output_dir "results/mncai/Meta-Llama-3.1-8B-Instruct-v0.3-32batch_0.05warmup_onlyarc_5e5" \
+--dataset_name "mncai/ai2_arc" \
+--chat_template_format "llama3.1" \
+--add_special_tokens False \
+--append_concat_token False \
+--splits "train,test" \
+--max_seq_len 8192 \
+--num_train_epochs 2 \
+--logging_steps 5 \
+--log_level "info" \
+--logging_strategy "steps" \
+--eval_strategy "steps" \
+--eval_steps 10 \
+--save_total_limit 2 \
+--bf16 True \
+--packing False \
+--learning_rate 5e-5 \
+--lr_scheduler_type "cosine" \
+--weight_decay 1e-5 \
+--warmup_ratio 0.05 \
+--max_grad_norm 1.0 \
+--per_device_train_batch_size 32 \
+--per_device_eval_batch_size 32 \
+--gradient_accumulation_steps 1 \
+--gradient_checkpointing True \
+--use_reentrant False \
+--dataset_text_field "content" \
+--use_flash_attn True \
+--optim paged_adamw_32bit \
+--report_to "wandb" \
+# --hub_model_id "mncai/Meta-Llama-3.1-8B-Instruct-v0.2" \
+# --push_to_hub True \
+# --hub_private_repo True \
+# --hub_strategy "end" \
+# --hub_token $HF_WRITE_TOKEN \
+
+# --save_strategy "steps" \
+# --save_steps 200 \
+# --use_peft_lora True \
+# --lora_r 8 \
+# --lora_alpha 16 \
+# --lora_dropout 0.1 \
+# --lora_target_modules "all-linear" \
+# --use_4bit_quantization False
+
+# Dataset
+# ,mncai/hellaswag,mncai/mbpp,mncai/openbookqa,mncai/winogrande,mncai/trivia_qa,mncai/KMMLU"