Buckets:
| # ============================================================== | |
| # RunPod Environment Setup Script | |
| # ============================================================== | |
| # Run this on a FRESH RunPod pod to set up everything from scratch. | |
| # It bootstraps code from HuggingFace Buckets, installs dependencies | |
| # (without breaking the preinstalled PyTorch), and downloads training data. | |
| # | |
| # BOOTSTRAP (run this first on a fresh pod): | |
| # export HF_TOKEN="hf_your_token_here" | |
| # export HF_REPO_ID="minhluanquach/unet-training-data" | |
| # curl -sSL https://raw.githubusercontent.com/YOUR_USER/Train_Advanced_Lipsync_Model/main/scripts/runpod_setup.sh | bash | |
| # | |
| # Or if you already have the code: | |
| # cd /workspace/Train_Advanced_Lipsync_Model | |
| # bash scripts/runpod_setup.sh | |
| # | |
| # Environment variables: | |
| # HF_TOKEN - HuggingFace token (required for private buckets) | |
| # HF_REPO_ID - Bucket ID (default: minhluanquach/unet-training-data) | |
| # SKIP_DATA - Set to "1" to skip data download (code + deps only) | |
| # ============================================================== | |
| set -e # Exit on error | |
| BUCKET_ID="${HF_REPO_ID:-minhluanquach/unet-training-data}" | |
| PROJECT_DIR="/workspace/Train_Advanced_Lipsync_Model" | |
| echo "============================================" | |
| echo " RunPod Setup for Advanced LipSync Model" | |
| echo "============================================" | |
| echo " Bucket: $BUCKET_ID" | |
| echo " Target: $PROJECT_DIR" | |
| echo "============================================" | |
| # ----------------------------------------------------------- | |
| # Step 1: Fix temp directory (RunPod's /tmp is tiny) | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[1/8] Fixing temp directory..." | |
| export TMPDIR=/workspace/tmp | |
| mkdir -p "$TMPDIR" | |
| echo " ✓ TMPDIR set to $TMPDIR" | |
| # ----------------------------------------------------------- | |
| # Step 2: System dependencies | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[2/8] Installing system dependencies..." | |
| apt-get update -qq | |
| apt-get install -y -qq ffmpeg libsndfile1 git-lfs tmux htop > /dev/null 2>&1 | |
| echo " ✓ ffmpeg, libsndfile, git-lfs, tmux, htop installed" | |
| # ----------------------------------------------------------- | |
| # Step 3: Upgrade huggingface_hub (needed for Buckets API) | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[3/8] Installing huggingface_hub..." | |
| pip install -U huggingface_hub hf-xet --no-cache-dir --quiet 2>/dev/null | |
| echo " ✓ huggingface_hub installed" | |
| # Login if token is set | |
| if [ -n "$HF_TOKEN" ]; then | |
| python -c "from huggingface_hub import login; login(token='$HF_TOKEN')" 2>/dev/null | |
| echo " ✓ Logged in to HuggingFace" | |
| else | |
| echo " ⚠ No HF_TOKEN set. Run: huggingface-cli login" | |
| fi | |
| # ----------------------------------------------------------- | |
| # Step 4: Bootstrap code from HuggingFace Bucket | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[4/8] Downloading code from HF Bucket..." | |
| mkdir -p "$PROJECT_DIR" | |
| python -c " | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| api.sync_bucket( | |
| source='hf://buckets/$BUCKET_ID', | |
| dest='$PROJECT_DIR', | |
| include=['src/**', 'configs/**', 'scripts/**', 'models/**', 'tests/**', | |
| 'pyproject.toml', 'README.md', 'PLAN.md'], | |
| ) | |
| print(' ✓ Code downloaded') | |
| " | |
| cd "$PROJECT_DIR" | |
| # ----------------------------------------------------------- | |
| # Step 5: Install Python deps WITHOUT breaking system PyTorch | |
| # ----------------------------------------------------------- | |
| # RunPod pods come with PyTorch + CUDA preinstalled. | |
| # We MUST NOT reinstall torch/torchvision/torchaudio. | |
| # We install each dependency with --no-deps to avoid pip | |
| # trying to uninstall/rollback system packages (which fails | |
| # with "OSError: [Errno 2] No such file or directory"). | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[5/8] Installing Python dependencies (preserving system PyTorch)..." | |
| PACKAGES=( | |
| # Config & utilities | |
| omegaconf einops tqdm pyyaml | |
| # Losses | |
| lpips | |
| # Audio processing | |
| scipy numba llvmlite librosa soundfile soxr audioread pooch sounddevice | |
| # HuggingFace ecosystem | |
| tokenizers transformers diffusers accelerate | |
| # Vision | |
| opencv-python-headless scikit-image Pillow | |
| # Data | |
| pandas joblib scikit-learn | |
| # Pydantic (for HF hub) | |
| pydantic typing-inspection annotated-types | |
| # Monitoring | |
| tensorboard wandb | |
| # CLI tools | |
| rich markdown-it-py mdurl typer click | |
| # Misc deps | |
| gitpython gitdb protobuf markdown absl-py grpcio | |
| imageio tifffile lazy_loader kiwisolver fonttools | |
| contourpy cycler msgpack matplotlib | |
| ) | |
| FAILED=() | |
| for pkg in "${PACKAGES[@]}"; do | |
| if pip install --no-cache-dir --no-deps "$pkg" --quiet 2>/dev/null; then | |
| : # success, silent | |
| else | |
| FAILED+=("$pkg") | |
| echo " ⚠ Failed: $pkg" | |
| fi | |
| done | |
| # Register project as editable package (no deps — already installed above) | |
| pip install -e . --no-deps --no-cache-dir --quiet 2>/dev/null | |
| if [ ${#FAILED[@]} -eq 0 ]; then | |
| echo " ✓ All packages installed successfully" | |
| else | |
| echo " ⚠ Failed packages (${#FAILED[@]}): ${FAILED[*]}" | |
| echo " Try manually: pip install --no-cache-dir --no-deps <package>" | |
| fi | |
| # ----------------------------------------------------------- | |
| # Step 6: Verify GPU setup | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[6/8] Verifying GPU setup..." | |
| python -c " | |
| import torch | |
| num_gpus = torch.cuda.device_count() | |
| print(f' PyTorch: {torch.__version__}') | |
| print(f' CUDA: {torch.version.cuda}') | |
| print(f' GPUs: {num_gpus}') | |
| for i in range(num_gpus): | |
| p = torch.cuda.get_device_properties(i) | |
| print(f' GPU {i}: {p.name} ({p.total_memory / 1024**3:.1f} GB)') | |
| print(f' NCCL: {torch.distributed.is_nccl_available()}') | |
| " | |
| # Quick import check | |
| python -c " | |
| import omegaconf, einops, lpips, diffusers, transformers, accelerate, librosa | |
| print(' ✓ All critical imports OK') | |
| " || echo " ⚠ Some imports failed — check package list above" | |
| # ----------------------------------------------------------- | |
| # Step 7: Download training data from HuggingFace Bucket | |
| # ----------------------------------------------------------- | |
| if [ "${SKIP_DATA}" = "1" ]; then | |
| echo "" | |
| echo "[7/8] Skipping data download (SKIP_DATA=1)" | |
| else | |
| echo "" | |
| echo "[7/8] Downloading training data from HF Bucket..." | |
| echo " This will download ~200 GB. Re-running skips existing files." | |
| echo "" | |
| # Checkpoint (~354 MB) | |
| echo " --- Downloading checkpoint ---" | |
| python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step checkpoint | |
| echo "" | |
| # Cache (~27 GB) | |
| echo " --- Downloading cache ---" | |
| python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step cache | |
| echo "" | |
| # Processed (~145 GB as tar archives → extract) | |
| echo " --- Downloading processed data ---" | |
| python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step processed | |
| echo "" | |
| fi | |
| # ----------------------------------------------------------- | |
| # Step 8: Verify everything | |
| # ----------------------------------------------------------- | |
| echo "" | |
| echo "[8/8] Verifying setup..." | |
| python scripts/hf/download_from_hf.py --repo-id "$BUCKET_ID" --step verify | |
| # --- Summary --- | |
| NUM_GPUS=$(python -c "import torch; print(torch.cuda.device_count())") | |
| echo "" | |
| echo "============================================" | |
| echo " Setup Complete!" | |
| echo "============================================" | |
| echo "" | |
| echo " GPUs detected: $NUM_GPUS" | |
| echo "" | |
| echo " To start training:" | |
| echo " ─────────────────" | |
| echo " # Main UNet model (distributed):" | |
| echo " torchrun --nproc_per_node=$NUM_GPUS \\" | |
| echo " -m src.training.train_model_distributed \\" | |
| echo " --config configs/train_distributed.yaml" | |
| echo "" | |
| echo " # Resume from checkpoint:" | |
| echo " torchrun --nproc_per_node=$NUM_GPUS \\" | |
| echo " -m src.training.train_model_distributed \\" | |
| echo " --config configs/train_distributed.yaml \\" | |
| echo " --resume checkpoints/best_model.pt" | |
| echo "" | |
| echo " # Monitor (in tmux or another terminal):" | |
| echo " tensorboard --logdir logs --bind_all --port 6006" | |
| echo "" | |
Xet Storage Details
- Size:
- 8.2 kB
- Xet hash:
- 2020d8934248573a0eb025955b87bb57b996168b86ee02b9b8fad0a97224a992
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.