File size: 1,905 Bytes
97c39f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #!/usr/bin/env bash
# Self-healing runner for the 528M full-corpus 16k re-encode.
# Loops: rm partial -> re-encode -> verify against the deterministic anchor
# (2 complete runs produced 11,544,766 non-empty lines / 520,133,183 tokens;
# empty EOT segments are dropped by reencode.py, so the retrain's 11,545,267
# line count is NOT the right expectation).
# Stops when the output file matches BOTH anchors.
# Launch detached: setsid nohup ./train/watchdog_reencode_full.sh > /dev/null 2>&1 < /dev/null &
set -u
cd "$(dirname "$0")/.."
export PYTHONPATH=$PWD
LOG=logs/tokenizer16k.log
EXPECT=11544766
TOK_EXPECT=520133183
for attempt in $(seq 1 99); do
echo "=== full re-encode attempt $attempt $(date "+%Y-%m-%d %H:%M:%S") ===" >> "$LOG"
rm -f data/train_full16k.bin
.venv/bin/python data/reencode.py \
--in data/train_full.bin --old-tok data/tokenizer.json \
--new-tok data/tokenizer16k.json --out data/train_full16k.bin >> "$LOG" 2>&1
code=$?
echo "=== attempt $attempt exit $code $(date "+%H:%M:%S") ===" >> "$LOG"
lines=$(.venv/bin/python -c "
import numpy as np
from data.tokenizer import load_tokenizer
tok = load_tokenizer('data/tokenizer16k.json')
eot = tok.token_to_id('<|endoftext|>')
mm = np.memmap('data/train_full16k.bin', dtype='<u2', mode='r')
n = len(mm)
cnt = 0
for start in range(0, n, 4_000_000):
cnt += int((mm[start:start + 4_000_000] == eot).sum())
print(cnt)
")
tokens=$(.venv/bin/python -c "import os; print(os.path.getsize('data/train_full16k.bin') // 2)")
echo "attempt $attempt lines=$lines expect=$EXPECT tokens=$tokens expect_tokens=$TOK_EXPECT" >> "$LOG"
if [ "$lines" -eq "$EXPECT" ] && [ "$tokens" -eq "$TOK_EXPECT" ]; then
echo "full re-encode COMPLETE ($lines lines, $tokens tokens)" >> "$LOG"
exit 0
fi
echo "incomplete (or killed); retrying in 10s" >> "$LOG"
sleep 10
done
echo "gave up after 99 attempts" >> "$LOG"
exit 1
|