experiment-2107/scripts/temporal_scripts/fs_spot_run_all.sh

#!/bin/bash
# Drive extraction to completion. Re-runs in passes so clips become available as
# their broadcast finishes downloading; each pass shards across NSHARD processes
# to saturate the GPU (one process tops out ~155fps, the 3090 does ~380fps).
cd /workspace/scripts_7/scripts
NSHARD=4
for pass in $(seq 1 40); do
  done_n=$(ls /workspace/scripts_7/data/fs_spot/processed/*.npz 2>/dev/null | wc -l)
  vids=$(ls /workspace/scripts_7/data/fs_spot/videos/*.webm 2>/dev/null | wc -l)
  echo "=== pass $pass | extracted ${done_n}/371 | videos ${vids}/11 ==="
  [ "$done_n" -ge 371 ] && { echo "ALL 371 EXTRACTED"; break; }
  pids=()
  for i in $(seq 0 $((NSHARD-1))); do
    python3 -m temporal_scripts.fs_spot_extract --shard $i/$NSHARD > /tmp/shard_$i.log 2>&1 &
    pids+=($!)
  done
  for p in "${pids[@]}"; do wait $p; done
  new_n=$(ls /workspace/scripts_7/data/fs_spot/processed/*.npz 2>/dev/null | wc -l)
  echo "    pass $pass added $((new_n-done_n)) clips -> ${new_n}/371"
  # nothing new and downloads still running -> wait for more video to land
  if [ "$new_n" -eq "$done_n" ] && [ "$vids" -lt 11 ]; then echo "    waiting for downloads..."; sleep 120; fi
done
echo "FINAL: $(ls /workspace/scripts_7/data/fs_spot/processed/*.npz 2>/dev/null | wc -l)/371"