scripts/hf_upload_mirror.py

"""Resumable large-folder upload to a Hugging Face mirror.

Works around two mirror-side issues with the LFS multipart-completion endpoint
(hf-mirror.org): an incomplete SSL chain and intermittent 504 gateway timeouts.
- SSL: inject a requests session with verify=False via configure_http_backend.
- 504: rely on upload_large_folder's built-in retry loop.
"""

import os
import warnings

import urllib3
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

warnings.filterwarnings("ignore")
urllib3.disable_warnings()

# NOTE: hf_transfer uses its own Rust HTTP client that ignores the verify=False
# session below, so it cannot bypass the mirror's broken SSL chain on the LFS
# multipart-completion endpoint. Keep it disabled here.
os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com")
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "0"

from huggingface_hub import configure_http_backend, HfApi


def _backend() -> requests.Session:
    session = requests.Session()
    session.verify = False
    # The mirror gateway intermittently returns 502/503/504 when completing the
    # multipart assembly of multi-GB LFS shards. Retry at the HTTP layer (parts
    # are already on S3) so the file is NOT re-uploaded from scratch.
    retry = Retry(
        total=10,
        connect=10,
        read=10,
        status=10,
        backoff_factor=2,
        status_forcelist=(429, 500, 502, 503, 504),
        allowed_methods=False,  # retry on all methods incl. POST/PUT
        respect_retry_after_header=True,
        raise_on_status=False,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    session.mount("http://", adapter)
    return session


configure_http_backend(backend_factory=_backend)

IGNORE_PATTERNS = [
    "runs/*",
    "data/*",
    "checkpoints_tmp/*",
    # Public base models — the mirror cannot complete multipart assembly of the
    # multi-GB shards. Users download these from the official HF repos (see README).
    "checkpoints/Wan-AI/*",
    "checkpoints/DiffSynth-Studio/*",
    "*/__pycache__/*",
    "*/.cache/*",
    "src/fastwam.egg-info/*",
    "*.log",
    "PyTorch",
    "gcc",
    ".git/*",
]

if __name__ == "__main__":
    api = HfApi()
    api.upload_large_folder(
        repo_id="jwfanDL/fastwam",
        folder_path=".",
        repo_type="model",
        ignore_patterns=IGNORE_PATTERNS,
        num_workers=1,
        print_report=True,
    )