train/watchdog_reencode_full.sh

#!/usr/bin/env bash
# Self-healing runner for the 528M full-corpus 16k re-encode.
# Loops: rm partial -> re-encode -> verify against the deterministic anchor
# (2 complete runs produced 11,544,766 non-empty lines / 520,133,183 tokens;
# empty EOT segments are dropped by reencode.py, so the retrain's 11,545,267
# line count is NOT the right expectation).
# Stops when the output file matches BOTH anchors.
# Launch detached:  setsid nohup ./train/watchdog_reencode_full.sh > /dev/null 2>&1 < /dev/null &
set -u
cd "$(dirname "$0")/.."
export PYTHONPATH=$PWD
LOG=logs/tokenizer16k.log
EXPECT=11544766
TOK_EXPECT=520133183

for attempt in $(seq 1 99); do
  echo "=== full re-encode attempt $attempt $(date "+%Y-%m-%d %H:%M:%S") ===" >> "$LOG"
  rm -f data/train_full16k.bin
  .venv/bin/python data/reencode.py \
    --in data/train_full.bin --old-tok data/tokenizer.json \
    --new-tok data/tokenizer16k.json --out data/train_full16k.bin >> "$LOG" 2>&1
  code=$?
  echo "=== attempt $attempt exit $code $(date "+%H:%M:%S") ===" >> "$LOG"
  lines=$(.venv/bin/python -c "
import numpy as np
from data.tokenizer import load_tokenizer
tok = load_tokenizer('data/tokenizer16k.json')
eot = tok.token_to_id('<|endoftext|>')
mm = np.memmap('data/train_full16k.bin', dtype='<u2', mode='r')
n = len(mm)
cnt = 0
for start in range(0, n, 4_000_000):
    cnt += int((mm[start:start + 4_000_000] == eot).sum())
print(cnt)
")
  tokens=$(.venv/bin/python -c "import os; print(os.path.getsize('data/train_full16k.bin') // 2)")
  echo "attempt $attempt lines=$lines expect=$EXPECT tokens=$tokens expect_tokens=$TOK_EXPECT" >> "$LOG"
  if [ "$lines" -eq "$EXPECT" ] && [ "$tokens" -eq "$TOK_EXPECT" ]; then
    echo "full re-encode COMPLETE ($lines lines, $tokens tokens)" >> "$LOG"
    exit 0
  fi
  echo "incomplete (or killed); retrying in 10s" >> "$LOG"
  sleep 10
done
echo "gave up after 99 attempts" >> "$LOG"
exit 1