04.pretrain_200k_base_5120.sh

#!/bin/bash
# Activate conda
source ~/miniconda3/etc/profile.d/conda.sh
conda activate bpe

# select GPU
export CUDA_VISIBLE_DEVICES=4

work_dir=/home/n5huang/dna_token
data_dir=${work_dir}/output_tokens
cache_dir=${data_dir}/cache

runnum=base_5120

# -p to use existing directory
mkdir -p ${work_dir}/pretrain/models/${runnum}
output_dir=${work_dir}/pretrain/models/${runnum}


cp /home/n5huang/dna-tokenizer/BPE_merge/04.pretrain_200k_base_5120.sh ${output_dir}
python /home/n5huang/dna-tokenizer/BPE_merge/run_mlm_200k.py \
					--output_dir ${output_dir} \
					--model_type bert \
					--tokenizer_name ${work_dir}/tokenizer_evaluation/baseline_bpe/vocab_5120/5120_tokenizer.json \
					--config_name ${work_dir}/pretrain/models/base_5120/config.json \
					--project_name token_eval_base_5120 \
					--do_train True \
					--model_max_length 512 \
					--max_seq_length 512 \
					--line_by_line True \
					--pad_to_max_length True \
					--train_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_train_chrOnly.tsv \
					--validation_file ${data_dir}/baseline_bpe_5120_allchr_all_tokenized_val_chrOnly.tsv \
					--cache_dir ${cache_dir} \
					--use_fast_tokenizer True \
					--do_eval True \
					--gradient_accumulation_steps 1 \
					--per_device_train_batch_size 96 \
					--per_device_eval_batch_size 96 \
					--save_steps 1000 \
					--save_total_limit 10 \
					--max_steps 200000 \
					--logging_steps 1000 \
					--learning_rate 4e-5 \
					--adam_epsilon 1e-6 \
					--weight_decay 0.01 \
					--adam_beta1 0.9 \
					--adam_beta2 0.98 \
					--mlm_probability 0.15 \
					--warmup_steps 10000 \
					--seed 42 \
					--preprocessing_num_workers 8 \
					--resume_from_checkpoint /home/n5huang/dna_token/pretrain/models/base_5120/checkpoint-100000