run.sh

#!/bin/bash
# Wire-Speed Training Launch Script
# Rust crawler -> tokenize -> Python transformer (live learning)

set -e
cd /workspace/wire_speed

echo "=== WIRE-SPEED TRANSFORMER ==="
echo "Building Rust feeder..."

# Install Rust if needed
if ! command -v cargo &> /dev/null; then
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
    source ~/.cargo/env
fi

# Build feeder
cd feeder
cargo build --release 2>&1 | tail -5
cd ..

echo "Feeder built!"
echo "Starting wire-speed training pipeline..."
echo "Rust (crawl+tokenize) | Python (train)"
echo ""

# Pipe: Rust outputs token IDs -> Python consumes and trains
./feeder/target/release/wire_feeder 2>/workspace/feeder.log | python3 stream_trainer.py 2>&1 | tee /workspace/wire_training.log