| export WANDB_MODE=disabled | |
| export CUDA_LAUNCH_BLOCKING=0 | |
| DATASET=Instruments | |
| BASE_MODEL=/datain/v-yinju/llama-7b | |
| CKPT_PATH=/datain/v-yinju/rq-llama/v3-train/Instruments/first | |
| DATA_PATH=/datain/v-yinju/rqvae-zzx/data | |
| OUTPUT_DIR=/datain/v-yinju/rq-llama/v3-train/Instruments/more_pretrain | |
| torchrun --nproc_per_node=8 --master_port=3324 continue_pretrain.py \ | |
| --base_model $BASE_MODEL \ | |
| --rqvae_model $CKPT_PATH \ | |
| --output_dir $OUTPUT_DIR \ | |
| --dataset $DATASET \ | |
| --data_path $DATA_PATH \ | |
| --per_device_batch_size 6 \ | |
| --gradient_accumulation_steps 2 \ | |
| --learning_rate 5e-5 \ | |
| --epochs 4 \ | |
| --weight_decay 0.01 \ | |
| --save_and_eval_strategy epoch \ | |
| --deepspeed ./config/ds_z2_fp16.json \ | |
| --dataloader_num_workers 4 \ | |
| --only_train_response \ | |
| --tasks seqrec,itemsearch,inters2title,inters2description,preferenceobtain,item2index,index2item,intertitles2item,query2item \ | |
| --train_prompt_sample_num 1,1,1,1,1,1,1,1,1 \ | |
| --train_data_sample_num 0,0,0,0,0,0,0,0,0 \ | |
| --fp16 &>>$OUTPUT_DIR/pretrain-log.txt | |
| cd convert | |
| nohup ./convert.sh $OUTPUT_DIR >convert.log 2>&1 & | |
| cd .. |