minhluanquach's picture
download
raw
8.2 kB
#!/bin/bash
# ==============================================================
# RunPod Environment Setup Script
# ==============================================================
# Run this on a FRESH RunPod pod to set up everything from scratch.
# It bootstraps code from HuggingFace Buckets, installs dependencies
# (without breaking the preinstalled PyTorch), and downloads training data.
#
# BOOTSTRAP (run this first on a fresh pod):
# export HF_TOKEN="hf_your_token_here"
# export HF_REPO_ID="minhluanquach/unet-training-data"
# curl -sSL https://raw.githubusercontent.com/YOUR_USER/Train_Advanced_Lipsync_Model/main/scripts/runpod_setup.sh | bash
#
# Or if you already have the code:
# cd /workspace/Train_Advanced_Lipsync_Model
# bash scripts/runpod_setup.sh
#
# Environment variables:
# HF_TOKEN - HuggingFace token (required for private buckets)
# HF_REPO_ID - Bucket ID (default: minhluanquach/unet-training-data)
# SKIP_DATA - Set to "1" to skip data download (code + deps only)
# ==============================================================
set -e # Exit on error
BUCKET_ID="${HF_REPO_ID:-minhluanquach/unet-training-data}"
PROJECT_DIR="/workspace/Train_Advanced_Lipsync_Model"
echo "============================================"
echo " RunPod Setup for Advanced LipSync Model"
echo "============================================"
echo " Bucket: $BUCKET_ID"
echo " Target: $PROJECT_DIR"
echo "============================================"
# -----------------------------------------------------------
# Step 1: Fix temp directory (RunPod's /tmp is tiny)
# -----------------------------------------------------------
echo ""
echo "[1/8] Fixing temp directory..."
export TMPDIR=/workspace/tmp
mkdir -p "$TMPDIR"
echo " ✓ TMPDIR set to $TMPDIR"
# -----------------------------------------------------------
# Step 2: System dependencies
# -----------------------------------------------------------
echo ""
echo "[2/8] Installing system dependencies..."
apt-get update -qq
apt-get install -y -qq ffmpeg libsndfile1 git-lfs tmux htop > /dev/null 2>&1
echo " ✓ ffmpeg, libsndfile, git-lfs, tmux, htop installed"
# -----------------------------------------------------------
# Step 3: Upgrade huggingface_hub (needed for Buckets API)
# -----------------------------------------------------------
echo ""
echo "[3/8] Installing huggingface_hub..."
pip install -U huggingface_hub hf-xet --no-cache-dir --quiet 2>/dev/null
echo " ✓ huggingface_hub installed"
# Login if token is set
if [ -n "$HF_TOKEN" ]; then
python -c "from huggingface_hub import login; login(token='$HF_TOKEN')" 2>/dev/null
echo " ✓ Logged in to HuggingFace"
else
echo " ⚠ No HF_TOKEN set. Run: huggingface-cli login"
fi
# -----------------------------------------------------------
# Step 4: Bootstrap code from HuggingFace Bucket
# -----------------------------------------------------------
echo ""
echo "[4/8] Downloading code from HF Bucket..."
mkdir -p "$PROJECT_DIR"
python -c "
from huggingface_hub import HfApi
api = HfApi()
api.sync_bucket(
source='hf://buckets/$BUCKET_ID',
dest='$PROJECT_DIR',
include=['src/**', 'configs/**', 'scripts/**', 'models/**', 'tests/**',
'pyproject.toml', 'README.md', 'PLAN.md'],
)
print(' ✓ Code downloaded')
"
cd "$PROJECT_DIR"
# -----------------------------------------------------------
# Step 5: Install Python deps WITHOUT breaking system PyTorch
# -----------------------------------------------------------
# RunPod pods come with PyTorch + CUDA preinstalled.
# We MUST NOT reinstall torch/torchvision/torchaudio.
# We install each dependency with --no-deps to avoid pip
# trying to uninstall/rollback system packages (which fails
# with "OSError: [Errno 2] No such file or directory").
# -----------------------------------------------------------
echo ""
echo "[5/8] Installing Python dependencies (preserving system PyTorch)..."
PACKAGES=(
# Config & utilities
omegaconf einops tqdm pyyaml
# Losses
lpips
# Audio processing
scipy numba llvmlite librosa soundfile soxr audioread pooch sounddevice
# HuggingFace ecosystem
tokenizers transformers diffusers accelerate
# Vision
opencv-python-headless scikit-image Pillow
# Data
pandas joblib scikit-learn
# Pydantic (for HF hub)
pydantic typing-inspection annotated-types
# Monitoring
tensorboard wandb
# CLI tools
rich markdown-it-py mdurl typer click
# Misc deps
gitpython gitdb protobuf markdown absl-py grpcio
imageio tifffile lazy_loader kiwisolver fonttools
contourpy cycler msgpack matplotlib
)
FAILED=()
for pkg in "${PACKAGES[@]}"; do
if pip install --no-cache-dir --no-deps "$pkg" --quiet 2>/dev/null; then
: # success, silent
else
FAILED+=("$pkg")
echo " ⚠ Failed: $pkg"
fi
done
# Register project as editable package (no deps — already installed above)
pip install -e . --no-deps --no-cache-dir --quiet 2>/dev/null
if [ ${#FAILED[@]} -eq 0 ]; then
echo " ✓ All packages installed successfully"
else
echo " ⚠ Failed packages (${#FAILED[@]}): ${FAILED[*]}"
echo " Try manually: pip install --no-cache-dir --no-deps <package>"
fi
# -----------------------------------------------------------
# Step 6: Verify GPU setup
# -----------------------------------------------------------
echo ""
echo "[6/8] Verifying GPU setup..."
python -c "
import torch
num_gpus = torch.cuda.device_count()
print(f' PyTorch: {torch.__version__}')
print(f' CUDA: {torch.version.cuda}')
print(f' GPUs: {num_gpus}')
for i in range(num_gpus):
p = torch.cuda.get_device_properties(i)
print(f' GPU {i}: {p.name} ({p.total_memory / 1024**3:.1f} GB)')
print(f' NCCL: {torch.distributed.is_nccl_available()}')
"
# Quick import check
python -c "
import omegaconf, einops, lpips, diffusers, transformers, accelerate, librosa
print(' ✓ All critical imports OK')
" || echo " ⚠ Some imports failed — check package list above"
# -----------------------------------------------------------
# Step 7: Download training data from HuggingFace Bucket
# -----------------------------------------------------------
if [ "${SKIP_DATA}" = "1" ]; then
echo ""
echo "[7/8] Skipping data download (SKIP_DATA=1)"
else
echo ""
echo "[7/8] Downloading training data from HF Bucket..."
echo " This will download ~200 GB. Re-running skips existing files."
echo ""
# Checkpoint (~354 MB)
echo " --- Downloading checkpoint ---"
python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step checkpoint
echo ""
# Cache (~27 GB)
echo " --- Downloading cache ---"
python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step cache
echo ""
# Processed (~145 GB as tar archives → extract)
echo " --- Downloading processed data ---"
python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step processed
echo ""
fi
# -----------------------------------------------------------
# Step 8: Verify everything
# -----------------------------------------------------------
echo ""
echo "[8/8] Verifying setup..."
python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step verify
# --- Summary ---
NUM_GPUS=$(python -c "import torch; print(torch.cuda.device_count())")
echo ""
echo "============================================"
echo " Setup Complete!"
echo "============================================"
echo ""
echo " GPUs detected: $NUM_GPUS"
echo ""
echo " To start training:"
echo " ─────────────────"
echo " # Main UNet model (distributed):"
echo " torchrun --nproc_per_node=$NUM_GPUS \\"
echo " -m src.training.train_model_distributed \\"
echo " --config configs/train_distributed.yaml"
echo ""
echo " # Resume from checkpoint:"
echo " torchrun --nproc_per_node=$NUM_GPUS \\"
echo " -m src.training.train_model_distributed \\"
echo " --config configs/train_distributed.yaml \\"
echo " --resume checkpoints/best_model.pt"
echo ""
echo " # Monitor (in tmux or another terminal):"
echo " tensorboard --logdir logs --bind_all --port 6006"
echo ""

Xet Storage Details

Size:
8.2 kB
·
Xet hash:
2020d8934248573a0eb025955b87bb57b996168b86ee02b9b8fad0a97224a992

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.