OpenTransformer commited on
Commit
fb56700
·
verified ·
1 Parent(s): 44d9388

Upload run.sh with huggingface_hub

Browse files
Files changed (1) hide show
  1. run.sh +28 -0
run.sh ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Wire-Speed Training Launch Script
3
+ # Rust crawler -> tokenize -> Python transformer (live learning)
4
+
5
+ set -e
6
+ cd /workspace/wire_speed
7
+
8
+ echo "=== WIRE-SPEED TRANSFORMER ==="
9
+ echo "Building Rust feeder..."
10
+
11
+ # Install Rust if needed
12
+ if ! command -v cargo &> /dev/null; then
13
+ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
14
+ source ~/.cargo/env
15
+ fi
16
+
17
+ # Build feeder
18
+ cd feeder
19
+ cargo build --release 2>&1 | tail -5
20
+ cd ..
21
+
22
+ echo "Feeder built!"
23
+ echo "Starting wire-speed training pipeline..."
24
+ echo "Rust (crawl+tokenize) | Python (train)"
25
+ echo ""
26
+
27
+ # Pipe: Rust outputs token IDs -> Python consumes and trains
28
+ ./feeder/target/release/wire_feeder 2>/workspace/feeder.log | python3 stream_trainer.py 2>&1 | tee /workspace/wire_training.log