scripts/auto_launch_language_training.sh

#!/bin/bash
# Auto-launch language training when embeddings are ready
# This script monitors embeddings and auto-submits training

set -euo pipefail

EMBEDDINGS_FILE="/scratch/$USER/dovla/experiments/instruction_embeddings.pkl"
MAX_WAIT=3600  # Wait max 1 hour

echo "======================================"
echo "AUTO-LAUNCH: Language Training"
echo "======================================"
echo ""
echo "Monitoring: $EMBEDDINGS_FILE"
echo "Max wait: ${MAX_WAIT}s (1 hour)"
echo ""

# Wait for embeddings to be ready
elapsed=0
while [ $elapsed -lt $MAX_WAIT ]; do
    if [ -f "$EMBEDDINGS_FILE" ]; then
        # Verify embeddings are complete
        if python3 -c "
import pickle
try:
    emb = pickle.load(open('$EMBEDDINGS_FILE', 'rb'))
    print(f'✅ Embeddings ready: {len(emb)} groups')
    exit(0 if len(emb) >= 3000 else 1)
except:
    exit(1)
" 2>/dev/null; then
            echo ""
            echo "✅ Embeddings verified!"
            echo ""

            # Launch language training
            echo "🚀 Launching language training (3 seeds)..."
            JOB_ID=$(sbatch scripts/slurm/train_transformer_lang.sbatch 2>&1 | grep -oP '\d+')

            echo "✅ Submitted: Job $JOB_ID"
            echo ""
            echo "Expected completion: 2-3 hours"
            echo "Expected result: 50-55% selected success"
            echo ""

            exit 0
        fi
    fi

    sleep 30
    elapsed=$((elapsed + 30))

    if [ $((elapsed % 300)) -eq 0 ]; then
        echo "Still waiting... (${elapsed}s elapsed)"
    fi
done

echo ""
echo "⏰ Timeout after ${MAX_WAIT}s"
echo "Manual launch required: sbatch scripts/slurm/train_transformer_lang.sbatch"
exit 1