File size: 1,920 Bytes
8eb4303 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | #!/usr/bin/env bash
set -e
# --- initialize conda ---
source /root/miniconda3/etc/profile.d/conda.sh
# --- activate env ---
conda activate bpe_v2
# select GPU
work_dir=/root/NaN/dna-tokenizer
data_dir=${work_dir}/dna-tokenizer-data
cache_dir=${data_dir}/cache
# runnum=len2_2048
runnum=base_5120
mkdir -p ${work_dir}/pretrain/models/model_${runnum}
output_dir=${work_dir}/pretrain/models/model_${runnum}
# export CUDA_VISIBLE_DEVICES=0
# export WORLD_SIZE=1
# export RANK=0
# export LOCAL_RANK=0
# export MASTER_ADDR=127.0.0.1
# export MASTER_PORT=29500
cp 04.pretrain.sh ${output_dir}
torchrun --nproc_per_node=8 run_mlm.py \
--output_dir ${output_dir} \
--model_type bert \
--tokenizer_name ${work_dir}/baseline_bpe/vocab_5120/5120_tokenizer.json \
--config_name ${work_dir}/BPE_merge/config_5120.json \
--project_name token_eval_base_5120 \
--do_train True \
--model_max_length 512 \
--max_seq_length 512 \
--line_by_line True \
--pad_to_max_length True \
--train_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
--validation_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
--cache_dir ${cache_dir} \
--use_fast_tokenizer True \
--do_eval True \
--gradient_accumulation_steps 1 \
--per_device_train_batch_size 96 \
--per_device_eval_batch_size 96 \
--save_steps 1000 \
--save_total_limit 10 \
--max_steps 100000 \
--logging_steps 1000 \
--learning_rate 4e-5 \
--adam_epsilon 1e-6 \
--weight_decay 0.01 \
--adam_beta1 0.9 \
--adam_beta2 0.98 \
--mlm_probability 0.15 \
--warmup_steps 10000 \
--seed 42 \
--preprocessing_num_workers 8 \
--overwrite_output_dir True
# --resume_from_checkpoint /root/NaN/dna-tokenizer/pretrain/models/model_base_2048/checkpoint-37000
|