File size: 1,683 Bytes
68442cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | #!/bin/bash
# Auto-launch language training when embeddings are ready
# This script monitors embeddings and auto-submits training
set -euo pipefail
EMBEDDINGS_FILE="/scratch/$USER/dovla/experiments/instruction_embeddings.pkl"
MAX_WAIT=3600 # Wait max 1 hour
echo "======================================"
echo "AUTO-LAUNCH: Language Training"
echo "======================================"
echo ""
echo "Monitoring: $EMBEDDINGS_FILE"
echo "Max wait: ${MAX_WAIT}s (1 hour)"
echo ""
# Wait for embeddings to be ready
elapsed=0
while [ $elapsed -lt $MAX_WAIT ]; do
if [ -f "$EMBEDDINGS_FILE" ]; then
# Verify embeddings are complete
if python3 -c "
import pickle
try:
emb = pickle.load(open('$EMBEDDINGS_FILE', 'rb'))
print(f'✅ Embeddings ready: {len(emb)} groups')
exit(0 if len(emb) >= 3000 else 1)
except:
exit(1)
" 2>/dev/null; then
echo ""
echo "✅ Embeddings verified!"
echo ""
# Launch language training
echo "🚀 Launching language training (3 seeds)..."
JOB_ID=$(sbatch scripts/slurm/train_transformer_lang.sbatch 2>&1 | grep -oP '\d+')
echo "✅ Submitted: Job $JOB_ID"
echo ""
echo "Expected completion: 2-3 hours"
echo "Expected result: 50-55% selected success"
echo ""
exit 0
fi
fi
sleep 30
elapsed=$((elapsed + 30))
if [ $((elapsed % 300)) -eq 0 ]; then
echo "Still waiting... (${elapsed}s elapsed)"
fi
done
echo ""
echo "⏰ Timeout after ${MAX_WAIT}s"
echo "Manual launch required: sbatch scripts/slurm/train_transformer_lang.sbatch"
exit 1
|