download_missing_models.sh: GITHUB_TOKEN auth + codeload tarball fallback for git clones (datacenter IPs get 403 from github.com)

download_missing_models.sh

@@ -20,6 +20,12 @@
 #
 # Auth: uses the HUGGING_FACE_ACCESS_TOKEN environment variable if set
 # (falls back to HF_TOKEN), otherwise prompts for a token.
+# If GITHUB_TOKEN (or GH_TOKEN) is set, git clones of github.com repos are
+# authenticated with it — datacenter IPs (e.g. RunPod) often get 403s on
+# anonymous github.com traffic. Use a fine-grained PAT with no repo
+# permissions; the token is sent as a one-shot header, never written to disk.
+# With or without a token, a failed clone falls back to a repo tarball from
+# codeload.github.com, which is typically not IP-blocked.
 
 set -euo pipefail
 
@@ -312,14 +318,82 @@ def resolve_repo_url(dep):
     return None, None
 
 GIT_ENV = {**os.environ, "GIT_TERMINAL_PROMPT": "0"}
+GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+
+def github_owner_repo(url):
+    m = re.match(r"https?://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", url)
+    return (m.group(1), m.group(2)) if m else None
+
+def gh_headers():
+    h = {"User-Agent": "comfy-dep-installer"}
+    if GITHUB_TOKEN:
+        h["Authorization"] = f"Bearer {GITHUB_TOKEN}"
+    return h
+
+def git_clone(url, dest):
+    """Clone, authenticating github.com with GITHUB_TOKEN when available.
+    The token goes in a per-invocation header (git -c), so it is never
+    persisted in the clone's .git/config."""
+    cmd = ["git"]
+    if GITHUB_TOKEN and github_owner_repo(url):
+        import base64
+        basic = base64.b64encode(f"x-access-token:{GITHUB_TOKEN}".encode()).decode()
+        cmd += ["-c", f"http.https://github.com/.extraheader=Authorization: basic {basic}"]
+    cmd += ["clone", "--recursive", "--progress", url, dest]
+    subprocess.run(cmd, check=True, env=GIT_ENV, timeout=900, stdin=subprocess.DEVNULL)
+
+def install_from_tarball(url, ver, dest):
+    """Fallback when git clone is refused (github.com 403s anonymous requests
+    from many datacenter IPs): fetch the repo tarball from codeload.github.com,
+    which sits on separate infrastructure and is typically not blocked.
+    Caveats vs a real clone: no .git (ComfyUI-Manager can't update the pack)
+    and no submodules."""
+    owner, repo = github_owner_repo(url)
+    if ver and re.fullmatch(r"[0-9a-f]{40}", str(ver), re.I):
+        ref = str(ver)
+    else:
+        info = fetch_json_gh(f"https://api.github.com/repos/{owner}/{repo}")
+        ref = info.get("default_branch") or "main"
+    print(f"  {C_YELLOW}falling back to tarball of {owner}/{repo}@{ref[:12]} via codeload.github.com ...{C_RESET}")
+    import shutil, tarfile, tempfile
+    req = urllib.request.Request(
+        f"https://codeload.github.com/{owner}/{repo}/tar.gz/{ref}", headers=gh_headers())
+    os.makedirs(CUSTOM_NODES_DIR, exist_ok=True)
+    # tmp dir lives next to dest so the final rename stays on one filesystem;
+    # "." prefix keeps installed_packs() from ever seeing it
+    with tempfile.TemporaryDirectory(dir=CUSTOM_NODES_DIR, prefix=".tarball-") as tmp:
+        tar_path = os.path.join(tmp, "repo.tar.gz")
+        with urllib.request.urlopen(req, timeout=120) as r, open(tar_path, "wb") as fh:
+            shutil.copyfileobj(r, fh)
+        with tarfile.open(tar_path) as tar:
+            try:
+                tar.extractall(tmp, filter="data")
+            except TypeError:  # Python < 3.12 has no filter=
+                tar.extractall(tmp)
+        tops = [e for e in os.listdir(tmp)
+                if os.path.isdir(os.path.join(tmp, e))]
+        if len(tops) != 1:
+            raise RuntimeError(f"unexpected tarball layout: {tops}")
+        os.rename(os.path.join(tmp, tops[0]), dest)
+
+def fetch_json_gh(url):
+    req = urllib.request.Request(url, headers=gh_headers())
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.load(r)
 
 def install_pack(name, url, ver):
     dest = os.path.join(CUSTOM_NODES_DIR, url.rstrip("/").split("/")[-1].removesuffix(".git"))
     print(f"\n{C_CYAN}Installing {name} from {url} ...{C_RESET}")
     if not os.path.isdir(dest):
-        subprocess.run(["git", "clone", "--recursive", "--progress", url, dest],
-                       check=True, env=GIT_ENV, timeout=900, stdin=subprocess.DEVNULL)
-    if ver and re.fullmatch(r"[0-9a-f]{40}", str(ver), re.I):
+        try:
+            git_clone(url, dest)
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            if not github_owner_repo(url):
+                raise
+            print(f"  {C_YELLOW}⚠ git clone failed ({e}){C_RESET}")
+            install_from_tarball(url, ver, dest)
+    if ver and re.fullmatch(r"[0-9a-f]{40}", str(ver), re.I) \
+            and os.path.isdir(os.path.join(dest, ".git")):
         r = subprocess.run(["git", "-C", dest, "checkout", ver], env=GIT_ENV, timeout=120,
                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         print(f"  pinned to commit {ver[:12]}" if r.returncode == 0