File size: 1,851 Bytes
e2bfccc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | #!/usr/bin/env bash
set -euo pipefail
RUN_ID="${RUN_ID:-taotern-pre-200m-stability-gate-$(date +%Y%m%d-%H%M%S)}"
JOB_ROOT="${JOB_ROOT:-/home/student/YouZheng/jobs/taotern}"
REMOTE_REPO="${REMOTE_REPO:-$(pwd)}"
PYTHON_BIN="${PYTHON_BIN:-/home/student/.venv/bin/python}"
SSM_REPO_PATH="${SSM_REPO_PATH:-/home/student/YouZheng/gamma_ssm_repo}"
DATA_PATH="${DATA_PATH:-/home/student/Data/TaoData/pretrain.jsonl}"
SFT_DATA_PATH="${SFT_DATA_PATH:-/home/student/Data/TaoData/sft.jsonl}"
TOKENIZER_PATH="${TOKENIZER_PATH:-/home/student/YouZheng/tokenizers/taodata_pilot_8k/tokenizer.model}"
SEQ_LEN="${SEQ_LEN:-512}"
BATCH_SIZE="${BATCH_SIZE:-8}"
TARGET_TOKENS="${TARGET_TOKENS:-20000000}"
MAX_TOKENS="${MAX_TOKENS:-50000000}"
MAX_RECORDS="${MAX_RECORDS:-120000}"
EVAL_BATCHES="${EVAL_BATCHES:-64}"
LEARNING_RATE="${LEARNING_RATE:-0.0008}"
WEIGHT_DECAY="${WEIGHT_DECAY:-0.01}"
TRAIN_LOG_EVERY="${TRAIN_LOG_EVERY:-250}"
SFT_SANITY_SAMPLES="${SFT_SANITY_SAMPLES:-4}"
SFT_SANITY_STEPS="${SFT_SANITY_STEPS:-120}"
SFT_SANITY_LR="${SFT_SANITY_LR:-0.00005}"
JOB_COMMAND="REMOTE_REPO=$REMOTE_REPO PYTHON_BIN=$PYTHON_BIN SSM_REPO_PATH=$SSM_REPO_PATH DATA_PATH=$DATA_PATH SFT_DATA_PATH=$SFT_DATA_PATH TOKENIZER_PATH=$TOKENIZER_PATH SEQ_LEN=$SEQ_LEN BATCH_SIZE=$BATCH_SIZE TARGET_TOKENS=$TARGET_TOKENS MAX_TOKENS=$MAX_TOKENS MAX_RECORDS=$MAX_RECORDS EVAL_BATCHES=$EVAL_BATCHES LEARNING_RATE=$LEARNING_RATE WEIGHT_DECAY=$WEIGHT_DECAY TRAIN_LOG_EVERY=$TRAIN_LOG_EVERY SFT_SANITY_SAMPLES=$SFT_SANITY_SAMPLES SFT_SANITY_STEPS=$SFT_SANITY_STEPS SFT_SANITY_LR=$SFT_SANITY_LR bash scripts/remote/run_pre_200m_stability_gate.sh"
export RUN_ID JOB_ROOT JOB_COMMAND
export OUTPUT_DIR="${OUTPUT_DIR:-$JOB_ROOT/$RUN_ID/outputs}"
export CHECKPOINT_DIR="${CHECKPOINT_DIR:-$JOB_ROOT/$RUN_ID/checkpoints}"
bash scripts/remote/submit_detached_job.sh
|