File size: 522 Bytes
fc56522 | 1 | /usr/local/bin/train_evo2 -d train_data_config.yml --dataset-dir ./preprocessed --result-dir pretraining_demo --experiment-name evo2 --model-size 1b --devices 8 --num-nodes 1 --seq-length 8192 --micro-batch-size 2 --lr 0.000015 --min-lr 0.0000149 --warmup-steps 100 --grad-acc-batches 4 --max-steps 1000 --ckpt-dir nemo2_evo2_1b_8k --clip-grad 250 --wd 0.001 --attention-dropout 0.01 --hidden-dropout 0.01 --val-check-interval 50 --activation-checkpoint-recompute-num-layers 5 --create-tensorboard-logger --ckpt-async-save |