File size: 598 Bytes
3cd1076 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | #!/bin/bash
DATA_DIR="YOUR_DATA_DIR"
python -u -m main \
loader.global_batch_size=512 \
loader.batch_size=32 \
loader.eval_batch_size=32 \
data=openwebtext-split \
data.cache_dir=$DATA_DIR \
wandb.project=owt_full \
wandb.name=owt_full_flm \
model=small \
algo=flm \
model.length=1024 \
sampling.num_sample_batches=1 \
sampling.solver=euler \
sampling.steps=[1024] \
trainer.max_steps=1500000 \
trainer.precision=bf16 \
optim.lr=3e-4 \
trainer.val_check_interval=5000 \
algo.double_temb=False \
callbacks.checkpoint_every_n_steps.every_n_train_steps=20000 \
|