| #SBATCH -A YOUR_ACCOUNT | |
| #SBATCH -p gpu | |
| #SBATCH --nodes 1 | |
| #SBATCH --gpus-per-node 1 | |
| #SBATCH --ntasks-per-node 1 | |
| #SBATCH -c 32 | |
| #SBATCH -t 03:00:00 | |
| #SBATCH --job-name=insc-evalfold | |
| #SBATCH -o logs/evalfold-%j.out | |
| #SBATCH -e logs/evalfold-%j.out | |
| # SLURM copies the submitted script into a per-job spool dir before running it on this | |
| # cluster, so locating the repo via ${BASH_SOURCE[0]} resolves to that spool path, not | |
| # the real one ("/var/lib/slurm/..." errors downstream) -- a real failure mode hit | |
| # repeatedly in practice. $SLURM_SUBMIT_DIR is set by sbatch to the directory it was | |
| # invoked from, immune to that copy, and matches this repo's own submit-from-root | |
| # convention (see scripts/slurm/README.md point 6). | |
| STOICHEIA_ROOT="${SLURM_SUBMIT_DIR:-$PWD}" | |
| export STOICHEIA_ROOT | |
| # Per-fold CER/top1/top20 on EACH fold's own held-out test digit, evaluated separately on | |
| # the inscriptions (iphi) and papyri test sets -- the un-conditioned baseline the new | |
| # metadata-primed models need to beat. Unlike insc_eval.sbatch (fixed digit 3/4), this | |
| # parameterizes test/val digit per fold so digit-3 records aren't wrongly treated as | |
| # held-out for folds whose OWN held-out digit is something else. | |
| # Usage: sbatch scripts/slurm/insc_eval_fold.sbatch <ckpt> <test_digit> <val_digit> <out_prefix> [n_per_length] | |
| # Contamination exclusion lists (per-digit, against the shared doc_clean pretraining | |
| # backbone -- see insc_leak_scan_fold.sbatch) are picked up automatically from | |
| # $INS_DATA/contaminated_test_fold<X>_docclean.json / contaminated_pap_test_fold<X>_docclean.json | |
| # if present; a missing file is a hard error, not a silent skip, so a fold can't | |
| # accidentally get scored uncontaminated-vs-contaminated against the others. | |
| set -euo pipefail | |
| CKPT="${1:?usage: sbatch scripts/slurm/insc_eval_fold.sbatch <ckpt> <test_digit> <val_digit> <out_prefix> [n]}" | |
| TEST_DIGIT="${2:?test_digit required}" | |
| VAL_DIGIT="${3:?val_digit required}" | |
| OUT_PREFIX="${4:?out_prefix required}" | |
| N="${5:-200}" | |
| INS_ROOT=$STOICHEIA_ROOT | |
| source "$INS_ROOT/env.sh" | |
| EXCL_IPHI="$INS_DATA/contaminated_test_fold${TEST_DIGIT}_docclean.json" | |
| EXCL_PAP="$INS_DATA/contaminated_pap_test_fold${TEST_DIGIT}_docclean.json" | |
| [ -f "$EXCL_IPHI" ] || { echo "MISSING $EXCL_IPHI -- run insc_leak_scan_fold.sbatch $TEST_DIGIT first" >&2; exit 1; } | |
| [ -f "$EXCL_PAP" ] || { echo "MISSING $EXCL_PAP -- run insc_leak_scan_fold.sbatch $TEST_DIGIT first" >&2; exit 1; } | |
| echo "ckpt=$CKPT test_digit=$TEST_DIGIT val_digit=$VAL_DIGIT excl_iphi=$EXCL_IPHI excl_pap=$EXCL_PAP $(date)" | |
| srun apptainer exec --nv $APPTAINER_BINDS $SIF bash -lc " | |
| set -e | |
| export PYTHONPATH=$STOICHEIA_ROOT | |
| export INSC_TEST_DIGIT=$TEST_DIGIT INSC_VAL_DIGIT=$VAL_DIGIT | |
| cd $INS_ROOT | |
| python insc/eval/restore.py --ckpt $CKPT --split test --domain iphi --n $N --exclude $EXCL_IPHI --out ${OUT_PREFIX}_iphi.json | |
| python insc/eval/restore.py --ckpt $CKPT --split test --domain papyri --n $N --exclude $EXCL_PAP --out ${OUT_PREFIX}_papyri.json | |
| " | |
| echo EVAL_FOLD_JOB_END | |