Upload mirror_all.py with huggingface_hub

mirror_all.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+import os, sys, tempfile, zipfile, time, traceback
+import requests, re
+from huggingface_hub import HfApi, create_repo, hf_hub_download
+
+HF_TOKEN = os.environ["HF_TOKEN"]
+CIVITAI_TOKEN = os.environ["CIVITAI_TOKEN"]
+HF_USER = "fwwrsd"
+WORK = "/workspace/mirror"
+os.makedirs(WORK, exist_ok=True)
+api = HfApi(token=HF_TOKEN)
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
+
+def log(msg):
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+def ensure_repo(name):
+    full = f"{HF_USER}/{name}"
+    try:
+        create_repo(repo_id=full, exist_ok=True, token=HF_TOKEN, private=False)
+        log(f"repo OK: {full}")
+    except Exception as e:
+        log(f"repo err (probably exists): {e}")
+    return full
+
+def civitai_download(version_id, dest_path):
+    url = f"https://civitai.com/api/download/models/{version_id}?token={CIVITAI_TOKEN}"
+    log(f"  CivitAI download v{version_id}")
+    with requests.get(url, stream=True, allow_redirects=True, headers={"User-Agent": UA}, timeout=300) as r:
+        r.raise_for_status()
+        tot = int(r.headers.get("Content-Length", 0))
+        log(f"    Content-Length: {tot/1e9:.2f} GB")
+        downloaded = 0
+        last_log = time.time()
+        with open(dest_path, "wb") as f:
+            for chunk in r.iter_content(chunk_size=8*1024*1024):
+                f.write(chunk)
+                downloaded += len(chunk)
+                if time.time() - last_log > 10:
+                    log(f"    {downloaded/1e9:.2f}/{tot/1e9:.2f} GB ({100*downloaded/max(tot,1):.0f}%)")
+                    last_log = time.time()
+    log(f"    DONE: {os.path.getsize(dest_path)/1e9:.2f} GB")
+
+def mediafire_download(mf_url, dest_path):
+    log(f"  MediaFire {mf_url}")
+    r = requests.get(mf_url, headers={"User-Agent": UA}, allow_redirects=True, timeout=60)
+    m = re.search(r'href="(https?://download\d+\.mediafire\.com/[^"]+)"', r.text)
+    if not m: raise RuntimeError("MediaFire: no direct URL")
+    direct = m.group(1)
+    log(f"    direct={direct}")
+    with requests.get(direct, headers={"User-Agent": UA}, stream=True, timeout=120) as rr:
+        rr.raise_for_status()
+        with open(dest_path, "wb") as f:
+            for chunk in rr.iter_content(chunk_size=1024*1024): f.write(chunk)
+    log(f"    DONE: {os.path.getsize(dest_path)/1e6:.1f} MB")
+
+def upload_to_hf(local_path, repo_full, name_in_repo):
+    log(f"  HF upload -> {repo_full}/{name_in_repo}")
+    api.upload_file(path_or_fileobj=local_path, path_in_repo=name_in_repo, repo_id=repo_full, token=HF_TOKEN)
+    log(f"    uploaded")
+
+def hf_mirror(src_repo, src_file, dest_repo, dest_file):
+    log(f"  HF mirror {src_repo}/{src_file} -> {dest_repo}/{dest_file}")
+    local = hf_hub_download(repo_id=src_repo, filename=src_file, token=HF_TOKEN, cache_dir="/tmp/hf_cache")
+    api.upload_file(path_or_fileobj=local, path_in_repo=dest_file, repo_id=dest_repo, token=HF_TOKEN)
+    os.unlink(local) if os.path.exists(local) else None
+    log(f"    uploaded")
+
+def step(num, total, name, fn):
+    log(f"")
+    log(f"=== [{num}/{total}] {name} ===")
+    try:
+        fn()
+        log(f"  STEP {num} OK")
+    except Exception as e:
+        log(f"  STEP {num} FAILED: {e}")
+        traceback.print_exc()
+
+def s1_lustify():
+    repo = ensure_repo("lustify-v7-ggwp")
+    p = os.path.join(WORK, "lustify_7.safetensors")
+    civitai_download(2155386, p)
+    upload_to_hf(p, repo, "lustify_7.safetensors")
+    os.unlink(p)
+
+def s2_donuts():
+    repo = ensure_repo("donuts-delivery-mix-v41")
+    p = os.path.join(WORK, "donutsdeliverymixV4_v41.safetensors")
+    civitai_download(1739105, p)
+    upload_to_hf(p, repo, "donutsdeliverymixV4_v41.safetensors")
+    os.unlink(p)
+
+def s3_detectors():
+    repo = ensure_repo("civitai-misc-detectors")
+    for vid, target in [(1309631, "assdetailer.pt"), (582143, "Eyeful_v2-Paired.pt")]:
+        z = os.path.join(WORK, f"{vid}.zip")
+        civitai_download(vid, z)
+        with zipfile.ZipFile(z) as zf:
+            pt_files = [m for m in zf.namelist() if m.endswith(".pt")]
+            if not pt_files: raise RuntimeError(f"No .pt in {z}: {zf.namelist()}")
+            zf.extract(pt_files[0], WORK)
+            upload_to_hf(os.path.join(WORK, pt_files[0]), repo, target)
+        os.unlink(z)
+
+def s4_skin():
+    repo = ensure_repo("openmodeldb-skin-upscalers")
+    for mf_url, name in [
+        ("https://www.mediafire.com/file/py46fnq12hhs5sz/1xSkinContrast-High-SuperUltraCompact.pth", "1xSkinContrast-High-SuperUltraCompact.pth"),
+        ("https://www.mediafire.com/file/5qwtnosdfuq9bgl/1xSkinContrast-HighAlternative-SuperUltraCompact.pth", "1xSkinContrast-HighAlternative-SuperUltraCompact.pth"),
+        ("https://www.mediafire.com/file/hnvatglitgayunh/1xSkinContrast-SuperUltraCompact.pth", "1xSkinContrast-SuperUltraCompact.pth"),
+    ]:
+        p = os.path.join(WORK, name)
+        try:
+            mediafire_download(mf_url, p)
+            upload_to_hf(p, repo, name)
+            os.unlink(p)
+        except Exception as e:
+            log(f"    skip {name}: {e}")
+
+def s5_gemma():
+    repo = ensure_repo("gitmylo-ltx2-gemma-mirror")
+    for f in [
+        "gemma_3_12B_it_fp8_e4m3fn.safetensors",
+        "gemma_3_12B_it_nvfp4_uncalibrated.safetensors",
+        "ltx-2-19b-dev-fp4_projections_only.safetensors",
+        "ltx-2-19b-dev-fp4_video_vae.safetensors",
+        "ltx-2-19b-dev-fp4_vocoder.safetensors",
+    ]:
+        try: hf_mirror("GitMylo/LTX-2-comfy_gemma_fp8_e4m3fn", f, repo, f)
+        except Exception as e: log(f"    skip {f}: {e}")
+
+log("=== mirror_all.py starting ===")
+log(f"HF user: {HF_USER}")
+log(f"Workspace: {WORK}")
+step(1, 5, "LUSTIFY v7", s1_lustify)
+step(2, 5, "Donuts v41", s2_donuts)
+step(3, 5, "NSFW detectors", s3_detectors)
+step(4, 5, "SkinContrast upscalers", s4_skin)
+step(5, 5, "GitMylo gemma mirror", s5_gemma)
+log("")
+log("=== ALL STEPS COMPLETE ===")