File size: 1,778 Bytes
8eb4303 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | #!/bin/bash
# Activate conda
source ~/miniconda3/etc/profile.d/conda.sh
conda activate bpe
# select GPU
export CUDA_VISIBLE_DEVICES=4
work_dir=/home/n5huang/dna_token
data_dir=${work_dir}/output_tokens
cache_dir=${data_dir}/cache
runnum=base_5120
# -p to use existing directory
mkdir -p ${work_dir}/pretrain/models/${runnum}
output_dir=${work_dir}/pretrain/models/${runnum}
cp /home/n5huang/dna-tokenizer/BPE_merge/04.pretrain_200k_base_5120.sh ${output_dir}
python /home/n5huang/dna-tokenizer/BPE_merge/run_mlm_200k.py \
--output_dir ${output_dir} \
--model_type bert \
--tokenizer_name ${work_dir}/tokenizer_evaluation/baseline_bpe/vocab_5120/5120_tokenizer.json \
--config_name ${work_dir}/pretrain/models/base_5120/config.json \
--project_name token_eval_base_5120 \
--do_train True \
--model_max_length 512 \
--max_seq_length 512 \
--line_by_line True \
--pad_to_max_length True \
--train_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
--validation_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
--cache_dir ${cache_dir} \
--use_fast_tokenizer True \
--do_eval True \
--gradient_accumulation_steps 1 \
--per_device_train_batch_size 96 \
--per_device_eval_batch_size 96 \
--save_steps 1000 \
--save_total_limit 10 \
--max_steps 200000 \
--logging_steps 1000 \
--learning_rate 4e-5 \
--adam_epsilon 1e-6 \
--weight_decay 0.01 \
--adam_beta1 0.9 \
--adam_beta2 0.98 \
--mlm_probability 0.15 \
--warmup_steps 10000 \
--seed 42 \
--preprocessing_num_workers 8 \
--resume_from_checkpoint /home/n5huang/dna_token/pretrain/models/base_5120/checkpoint-100000
|