File size: 739 Bytes
f0975ca | 1 2 3 4 5 6 7 8 9 10 11 | #!/bin/zsh
# Usage: run_main.sh [EFFORT] [MAX_TOKENS] [--with-eval]
# Generates the train split until the daily output-token budget hits the reserve.
# --with-eval first generates teacher references for the held-out eval prompts.
cd "${0:A:h}/.."
e=${1:-medium}; mt=${2:-16384}
if [ "$3" = "--with-eval" ]; then
.venv/bin/python -I scripts/generate.py --pool data/pool.jsonl --ids data/eval_ids.txt --out data/gen/eval.jsonl --effort $e --max-tokens $mt --concurrency 20 --per-minute 55 --log-every 50 --reserve-out 50000
fi
.venv/bin/python -I scripts/generate.py --pool data/pool.jsonl --ids data/train_ids.txt --out data/gen/train.jsonl --effort $e --max-tokens $mt --concurrency 20 --per-minute 55 --log-every 50 --reserve-out 50000
|