inti

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+from_url_spike_spiegel1.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,14 +1,99 @@
----
-title: SD Dataset Automaker2
-emoji: 😻
-colorFrom: green
-colorTo: red
-sdk: gradio
-sdk_version: 6.0.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: Generate anime character datasets from name or URLs
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: "SD Dataset Automaker: Fancaps → YOLO → Siamese → WD Tag → ZIP"
+emoji: 🎴💆📦
+colorFrom: indigo
+colorTo: pink
+sdk: gradio
+sdk_version: 6.0.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+# 🎴 SD Dataset Automaker
+
+Anime character dataset generator for LoRA/fine-tuning.
+
+## ⚡ Quick Start
+
+1. Find show on [fancaps.net](https://fancaps.net) → copy URL
+2. Upload 3-5 reference faces of target character
+3. Run → download ZIP with images + captions
+
+## 🎛️ Parameters
+
+| Param | Default | Description |
+|-------|---------|-------------|
+| URL | — | `fancaps.net/.../showimages.php?ID-Name` |
+| Refs | — | 3-5 face images of target character |
+| Character | auto | Output name & tag (optional) |
+| Max Downloads | 200 | Frames to scrape |
+| Similarity | 32 | Lower = stricter (25-30 strict, 35+ loose) |
+| Trigger Word | — | Prepends to all captions |
+
+## 🔌 MCP Server
+
+[![MCP](https://img.shields.io/badge/MCP-enabled-ff69b4)](https://huggingface.co/docs/hub/spaces-mcp-servers)
+
+**SSE:** `https://nekochu-sd-dataset-automaker.hf.space/gradio_api/mcp/sse`
+
+<details>
+<summary>Claude Desktop config</summary>
+
+```json
+{
+  "mcpServers": {
+    "sd-dataset-automaker": {
+      "command": "npx",
+      "args": ["mcp-remote", "https://nekochu-sd-dataset-automaker.hf.space/gradio_api/mcp/sse"]
+    }
+  }
+}
+```
+</details>
+
+<details>
+<summary>Python API (Gallery)</summary>
+
+```python
+from gradio_client import Client, handle_file
+
+client = Client("Nekochu/SD-Dataset-Automaker")
+zip_path, log = client.predict(
+    q="https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop",
+    c="spike_spiegel",
+    imgs=[
+        {"image": handle_file("https://huggingface.co/spaces/Nekochu/SD-Dataset-Automaker/resolve/main/from_url_spike_spiegel1.jpg"), "caption": None},
+        {"image": handle_file("https://huggingface.co/spaces/Nekochu/SD-Dataset-Automaker/resolve/main/from_url_spike_spiegel2.jpg"), "caption": None},
+    ],
+    files=None, mi=200, th=32, at="", an=True, mo=False, tv=False,
+    api_name="/process"
+)
+print(f"ZIP: {zip_path}")
+```
+</details>
+
+<details>
+<summary>Python API (File - MCP compatible)</summary>
+
+```python
+from gradio_client import Client, handle_file
+
+client = Client("Nekochu/SD-Dataset-Automaker")
+zip_path, log = client.predict(
+    q="https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop",
+    c="spike_spiegel",
+    imgs=None,  # Skip Gallery
+    files=[  # Use File input (simpler schema, MCP-friendly)
+        handle_file("spike_ref1.jpg"),
+        handle_file("spike_ref2.jpg"),
+    ],
+    mi=200, th=32, at="", an=True, mo=False, tv=False,
+    api_name="/process"
+)
+print(f"ZIP: {zip_path}")
+```
+</details>
+
+---
+*CPU runtime ~5-10 min*

app.py

@@ -0,0 +1,692 @@
+#!/usr/bin/env python3
+"""SD Dataset Automaker - HF Space (CPU) - Anime character dataset generator for LoRA/fine-tuning."""
+
+import warnings
+warnings.filterwarnings('ignore', category=FutureWarning)  # Suppress torch.cuda.amp spam
+warnings.filterwarnings('ignore', category=UserWarning, message='.*trust_repo.*')
+
+import os, re, shutil, zipfile, random, tempfile, argparse, sys
+from urllib.parse import quote_plus
+from collections import Counter
+from pathlib import Path
+
+from tqdm import tqdm
+import gradio as gr
+from bs4 import BeautifulSoup
+import requests as req_lib
+import time
+import numpy as np
+from PIL import Image
+import torch
+import torch.nn as nn
+from torchvision import models, transforms
+from sklearn.metrics.pairwise import pairwise_distances
+import onnxruntime as rt
+import pandas as pd
+import huggingface_hub
+
+# =============================================================================
+# CONFIG
+# =============================================================================
+EXTS = ('.jpg', '.jpeg', '.png')
+MODEL_DIR = Path(__file__).parent.resolve()  # Ensure absolute path
+YOLO_PATH = MODEL_DIR / "yolov5s_anime.pt"
+SIM_PATH = MODEL_DIR / "similarity.pt"
+EXAMPLES = [str(MODEL_DIR / f"from_url_spike_spiegel{i}.jpg") for i in range(1, 4)]  # absolute paths for gr.Examples
+WD_REPO = "SmilingWolf/wd-swinv2-tagger-v3"
+TAG_THRESH, BLACKLIST = 0.35, ["bangs", "breasts", "multicolored hair", "gradient hair", "two-tone hair", "virtual youtuber"]
+FACE_CONF, FACE_IOU, MIN_FACE, CROP_PAD = 0.5, 0.5, 35, 0.2
+SIM_THRESH, BATCH_SZ, FACE_SZ = 32, 16, 224
+
+# =============================================================================
+# UTILS
+# =============================================================================
+sanitize = lambda s: re.sub(r'[^\w.-]', '', s.replace(" ", "_"))
+get_imgs = lambda d: sorted([os.path.join(r,f) for r,_,fs in os.walk(d) for f in fs if f.lower().endswith(EXTS)])
+valid_img = lambda p: (lambda i: i.load() or True)(Image.open(p)) if os.path.exists(p) else False
+
+# HTTP session - mode depends on environment
+# CLI (local Windows): cloudscraper bypasses Cloudflare
+# HF Spaces: plain requests (cloudscraper fingerprint gets blocked on datacenter IPs)
+def init_session(use_cloudscraper=False):
+    global SESSION, HTTP_CLIENT
+    if use_cloudscraper:
+        try:
+            import cloudscraper
+            SESSION = cloudscraper.create_scraper()
+            HTTP_CLIENT = "cloudscraper"
+            return
+        except ImportError:
+            pass  # fallback to requests
+    SESSION = req_lib.Session()
+    SESSION.headers.update({
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'DNT': '1',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Cache-Control': 'max-age=0',
+        'Referer': 'https://fancaps.net/',
+    })
+    HTTP_CLIENT = "requests/browser-headers"
+
+# Default: plain requests (for HF Spaces import)
+SESSION = None
+HTTP_CLIENT = None
+init_session(use_cloudscraper=False)
+
+# =============================================================================
+# SCRAPING
+# =============================================================================
+def search_fancaps(prompt, anime=True, movies=False, tv=False, log_fn=None):
+    L = log_fn or print
+    url = f"https://fancaps.net/search.php?q={quote_plus(prompt)}&submit=Submit"
+    if anime: url += "&animeCB=Anime"
+    if movies: url += "&MoviesCB=Movies"
+    if tv: url += "&TVCB=TV"
+    L(f"  URL: {url}")
+    try:
+        resp = SESSION.get(url, timeout=30)
+        L(f"  Status: {resp.status_code}, Size: {len(resp.content)} bytes")
+        # Log key headers for debugging
+        cf_ray = resp.headers.get('cf-ray', 'none')
+        server = resp.headers.get('server', 'unknown')
+        L(f"  Server: {server}, CF-Ray: {cf_ray}")
+        if resp.status_code != 200:
+            L(f"  ERROR: HTTP {resp.status_code}")
+            # Show snippet of response to understand the block reason
+            content_snippet = resp.text[:500].replace('\n', ' ').strip()
+            L(f"  Response snippet: {content_snippet[:200]}...")
+            return {}
+        soup = BeautifulSoup(resp.content, "html.parser")
+        # Debug: check if we got Cloudflare challenge
+        title = soup.title.string if soup.title else "No title"
+        L(f"  Page title: {title[:50]}")
+        if "cloudflare" in title.lower() or "challenge" in title.lower():
+            L("  ERROR: Cloudflare challenge detected!")
+            return {}
+    except Exception as e:
+        L(f"  ERROR: {type(e).__name__}: {e}")
+        return {}
+    results, cnt = {}, 1
+    divs = soup.find_all("div", class_="single_post_content")
+    L(f"  Found {len(divs)} content divs")
+    for div in divs:
+        if not div.find('h2'): continue
+        for h2 in div.find_all('h2'):
+            title = h2.get_text(strip=True).lower()
+            cat = 'Movies' if 'movie' in title else 'TV' if 'tv' in title else 'Anime' if 'anime' in title else None
+            if not cat: continue
+            table = h2.find_next('table')
+            if not table: continue
+            results.setdefault(cat, [])
+            for h4 in table.find_all('h4'):
+                a = h4.find('a')
+                if a and a.get('href'):
+                    results[cat].append((a.get_text(strip=True), a['href'], cnt)); cnt += 1
+        break
+    L(f"  Parsed results: {sum(len(v) for v in results.values())} items in {list(results.keys())}")
+    return results
+
+def get_episodes(url, log_fn=None):
+    L = log_fn or (lambda x: None)
+    links, page = [], 1
+    while True:
+        try:
+            resp = SESSION.get(f"{url}&page={page}", timeout=20)
+            L(f"    get_episodes page {page}: status={resp.status_code}")
+            if resp.status_code != 200:
+                L(f"    ERROR: {resp.text[:150]}...")
+                break
+            soup = BeautifulSoup(resp.content, "html.parser")
+        except Exception as e:
+            L(f"    get_episodes ERROR: {type(e).__name__}: {e}")
+            break
+        btns = soup.find_all('a', class_='btn btn-block')
+        if not btns:
+            L(f"    No episode buttons on page {page}")
+            break
+        links.extend([("https://fancaps.net" + b['href'] if b['href'].startswith('/') else b['href']) for b in btns if b.get('href')])
+        L(f"    Found {len(btns)} episodes on page {page}, total: {len(links)}")
+        page += 1
+    return links or [url]
+
+def get_frame_names(url, log_fn=None):
+    L = log_fn or (lambda x: None)
+    names, page = [], 1
+    while True:
+        try:
+            resp = SESSION.get(f"{url}&page={page}", timeout=20)
+            if resp.status_code != 200:
+                L(f"    get_frames page {page}: status={resp.status_code}")
+                break
+            soup = BeautifulSoup(resp.content, "html.parser")
+        except Exception as e:
+            L(f"    get_frames ERROR: {type(e).__name__}: {e}")
+            break
+        imgs = soup.find_all('img', class_='imageFade')
+        if not imgs:
+            if page == 1: L(f"    No images found on first page!")
+            break
+        names.extend([s.split('/')[-1] for i in imgs if (s := i.get('src')) and s.split('/')[-1] not in names])
+        pager = soup.select_one('ul.pagination li:last-child a')
+        if not pager or pager.get('href') in ['#', None]: break
+        page += 1
+    L(f"    Total frame names: {len(names)}")
+    return names
+
+def download(url, folder, name, timeout=10, retries=3):
+    """Download single image with retry - returns (success, status_code)."""
+    fp = os.path.join(folder, name)
+    if os.path.exists(fp): return True, 200
+    for attempt in range(retries):
+        try:
+            r = SESSION.get(url, stream=True, timeout=timeout)
+            if r.status_code == 200:
+                with open(fp, 'wb') as f:
+                    for chunk in r.iter_content(16384):
+                        if chunk: f.write(chunk)
+                return True, 200
+            if r.status_code == 429:  # Rate limit - don't retry immediately
+                return False, 429
+            # Other errors - retry
+        except:
+            pass
+        if attempt < retries - 1:
+            time.sleep(1)
+    return False, None
+
+def scrape(name, link, save_dir, max_imgs, progress=None, log_fn=None):
+    L = log_fn or print
+    url, folder = "https://fancaps.net" + link, os.path.join(save_dir, sanitize(name))
+    os.makedirs(folder, exist_ok=True)
+    section = 'movie' if '/movies/' in link else 'anime' if '/anime/' in link else 'tv'
+    L(f"  [2/8] Scraping: {url}")
+    L(f"    Section: {section}, max: {max_imgs}")
+    consecutive_429 = 0
+    max_429 = 3  # Abort after 3 consecutive 429s
+
+    if section == 'movie':
+        names = get_frame_names(url, log_fn=L)
+        L(f"    Movie frames: {len(names)}")
+        sampled = random.sample(names, min(max_imgs, len(names))) if names else []
+        downloaded = 0
+        for i, n in enumerate(sampled):
+            if consecutive_429 >= max_429:
+                L(f"    Aborting: {consecutive_429} consecutive 429s")
+                break
+            if i > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
+            try:
+                if progress and len(sampled) > 0: progress((i+1)/len(sampled), desc=f"Downloading {name[:20]}")
+            except: pass
+            success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", folder, n)
+            if success:
+                downloaded += 1
+                consecutive_429 = 0
+            elif status == 429:
+                consecutive_429 += 1
+                cooldown = 30 * consecutive_429
+                L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
+                time.sleep(cooldown)
+            else:
+                consecutive_429 = 0
+        L(f"    Downloaded: {downloaded}/{len(sampled)}")
+    else:
+        L(f"    Fetching episodes...")
+        eps = get_episodes(url, log_fn=L)
+        L(f"    Episodes: {len(eps)}")
+        total = 0
+        per_ep = max(1, max_imgs // len(eps)) if eps else max_imgs
+        for i, ep in enumerate(eps):
+            if total >= max_imgs or consecutive_429 >= max_429: break
+            names = get_frame_names(ep, log_fn=L)
+            if not names: continue
+            ep_dir = os.path.join(folder, f"Ep{i+1}")
+            os.makedirs(ep_dir, exist_ok=True)
+            sampled = random.sample(names, min(per_ep, len(names), max_imgs - total))
+            for j, n in enumerate(sampled):
+                if consecutive_429 >= max_429: break
+                if j > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
+                try:
+                    if progress and max_imgs > 0: progress(total/max_imgs, desc=f"Ep{i+1}")
+                except: pass  # Gradio progress can fail in some contexts
+                success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", ep_dir, n)
+                if success:
+                    total += 1
+                    consecutive_429 = 0
+                elif status == 429:
+                    consecutive_429 += 1
+                    cooldown = 30 * consecutive_429
+                    L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
+                    time.sleep(cooldown)
+                else:
+                    consecutive_429 = 0
+        L(f"    Total downloaded: {total}")
+
+# =============================================================================
+# ML MODELS (cached)
+# =============================================================================
+_models = {}
+
+def get_yolo():
+    if 'yolo' not in _models:
+        _models['yolo'] = torch.hub.load('ultralytics/yolov5', 'custom', path=str(YOLO_PATH), force_reload=False, verbose=False)
+        _models['yolo'].conf, _models['yolo'].iou = FACE_CONF, FACE_IOU
+    return _models['yolo']
+
+def get_sim():
+    if 'sim' not in _models:
+        class SiameseNetwork(nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.base_model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)
+            def forward(self, x):
+                return self.base_model(x)  # 1000-class output (trained this way)
+        m = SiameseNetwork()
+        # Keep on CPU for consistent distance values across devices
+        m.load_state_dict(torch.load(str(SIM_PATH), map_location="cpu", weights_only=True))
+        m.eval()
+        _models['sim'] = m
+    return _models['sim']
+
+def get_tagger():
+    if 'tag' not in _models:
+        mp, cp = huggingface_hub.hf_hub_download(WD_REPO, "model.onnx"), huggingface_hub.hf_hub_download(WD_REPO, "selected_tags.csv")
+        tags = [str(x).replace('_', ' ') for x in pd.read_csv(cp)['name'].tolist()]
+        sess = rt.InferenceSession(mp, providers=['CPUExecutionProvider'])
+        _models['tag'] = (sess, tags, sess.get_inputs()[0].shape[1])
+    return _models['tag']
+
+# =============================================================================
+# PROCESSING
+# =============================================================================
+def dedup(paths, thresh=0.98):
+    if not paths: return [], []
+    m = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1); m.fc = nn.Identity(); m.eval()
+    tf = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([.485,.456,.406],[.229,.224,.225])])
+    emb, valid = [], []
+    with torch.no_grad():
+        for i in range(0, len(paths), 32):
+            batch = [(tf(Image.open(p).convert('RGB')), p) for p in paths[i:i+32] if valid_img(p)]
+            if batch:
+                x = torch.stack([b[0] for b in batch])
+                emb.append(m(x).numpy()); valid.extend([b[1] for b in batch])
+    del m
+    if not emb: return [], []
+    emb = np.vstack(emb); emb /= np.linalg.norm(emb, axis=1, keepdims=True).clip(1e-8)
+    sim = emb @ emb.T; np.fill_diagonal(sim, 0)
+    keep, drop = [], set()
+    for i in range(len(valid)):
+        if i not in drop: keep.append(valid[i]); drop.update(j for j in np.where(sim[i] > thresh)[0] if j > i)
+    return keep, [valid[i] for i in drop]
+
+def detect_faces(paths, out_dir):
+    yolo = get_yolo(); os.makedirs(out_dir, exist_ok=True); cnt = 0
+    for p in paths:
+        try:
+            img = Image.open(p).convert('RGB'); w, h = img.size
+            for j, det in enumerate(yolo(img, size=640).xyxy[0].cpu().numpy()):
+                x1, y1, x2, y2, conf, _ = det
+                bw, bh = x2-x1, y2-y1
+                x1, y1, x2, y2 = max(0, x1-bw*CROP_PAD), max(0, y1-bh*CROP_PAD), min(w, x2+bw*CROP_PAD), min(h, y2+bh*CROP_PAD)
+                if min(x2-x1, y2-y1) >= MIN_FACE:
+                    img.crop((int(x1), int(y1), int(x2), int(y2))).save(os.path.join(out_dir, f"{Path(p).stem}-{j+1}-{conf:.2f}.jpg"), quality=95)
+                    cnt += 1
+        except: pass
+    return cnt
+
+def face_emb(paths):
+    if not paths: return np.array([]), []
+    m = get_sim()  # Always on CPU for consistent distances
+    def pad(img):
+        t, w, h = FACE_SZ, img.size[0], img.size[1]; r = w/h
+        nw, nh = (t, int(t/r)) if r > 1 else (int(t*r), t)
+        img = img.resize((nw, nh), Image.BICUBIC)
+        out = Image.new('RGB', (t, t), (0,0,0)); out.paste(img, ((t-nw)//2, (t-nh)//2)); return out
+    tf = transforms.Compose([lambda x: pad(x.convert('RGB') if x.mode == 'RGBA' else x), transforms.ToTensor()])
+    emb, valid = [], []
+    with torch.no_grad():
+        for i in range(0, len(paths), BATCH_SZ):
+            batch = [(tf(Image.open(p)), p) for p in paths[i:i+BATCH_SZ] if valid_img(p)]
+            if batch:
+                x = torch.stack([b[0] for b in batch])  # CPU tensor
+                emb.append(m(x).numpy())
+                valid.extend([b[1] for b in batch])
+    return (np.vstack(emb), valid) if emb else (np.array([]), [])
+
+def tag(path, act_tag="", char_tag=""):
+    sess, tags, sz = get_tagger()
+    img = Image.open(path).convert('RGB'); w, h = img.size
+    s = min(sz/w, sz/h); nw, nh = int(w*s), int(h*s)
+    pad = Image.new('RGB', (sz, sz), (255,255,255)); pad.paste(img.resize((nw, nh), Image.BICUBIC), ((sz-nw)//2, (sz-nh)//2))
+    probs = sess.run(None, {sess.get_inputs()[0].name: np.expand_dims(np.array(pad).astype(np.float32)[:,:,::-1], 0)})[0][0]
+    found = [tags[i] for i, p in enumerate(probs) if p > TAG_THRESH and tags[i] not in BLACKLIST]
+    # Prepend activation tag and character tag if provided
+    prefix = []
+    if act_tag: prefix.append(act_tag); found = [t for t in found if t != act_tag]
+    if char_tag: prefix.append(char_tag.replace("_", " ")); found = [t for t in found if t != char_tag.replace("_", " ")]
+    return prefix + found
+
+# =============================================================================
+# PIPELINE
+# =============================================================================
+def parse_direct_url(url):
+    """Parse direct fancaps URL, extract show name and relative link. Returns (name, link) or (None, None)."""
+    # Match patterns: showimages.php?ID-Name, MovieImages.php?movieid=ID&name=Name, episodeimages.php?ID-Name
+    patterns = [
+        r'fancaps\.net/anime/showimages\.php\?(\d+)-([^&/]+)',      # anime show
+        r'fancaps\.net/tv/showimages\.php\?(\d+)-([^&/]+)',         # tv show
+        r'fancaps\.net/movies/MovieImages\.php\?.*?movieid=(\d+)',  # movie
+    ]
+    for pat in patterns:
+        m = re.search(pat, url)
+        if m:
+            if 'anime' in url: section = 'anime'
+            elif 'movies' in url: section = 'movies'
+            else: section = 'tv'
+            # Extract name from URL (replace underscores with spaces)
+            name = m.group(2).replace('_', ' ') if len(m.groups()) > 1 else f"Show_{m.group(1)}"
+            # Build relative link (what scrape() expects)
+            if section == 'movies':
+                link = f"/movies/MovieImages.php?movieid={m.group(1)}"
+            else:
+                link = f"/{section}/showimages.php?{m.group(1)}-{m.group(2) if len(m.groups()) > 1 else ''}"
+            return name, link
+    return None, None
+
+def run(query, char, examples, max_img, thresh, act_tag, anime, movies, tv, progress=None, cli_mode=False):
+    log = []
+    def L(m): log.append(m); print(m)
+    def prog(val, desc=""):
+        if progress and not cli_mode: progress(val, desc=desc)
+
+    work = tempfile.mkdtemp(prefix="ds_")
+    dirs = {k: os.path.join(work, f"{i}_{k}") for i, k in enumerate(['scrapped','filtered','faces','ex_faces','similar','results'], 1)}
+    for d in dirs.values(): os.makedirs(d, exist_ok=True)
+    final_zip = None  # Track ZIP for cleanup
+
+    try:
+        L(f"HTTP client: {HTTP_CLIENT}")
+        t0 = time.time()
+
+        # Check if query is a direct fancaps URL (bypasses search, works on HF Spaces)
+        if 'fancaps.net' in query and ('showimages.php' in query or 'MovieImages.php' in query):
+            L(f"[1/8] Direct URL mode")
+            name, link = parse_direct_url(query)
+            if not link:
+                return None, "\n".join(log) + "\n\nCouldn't parse URL!"
+            item = (name, link, 1)
+            L(f"  Parsed: {name}")
+        else:
+            L(f"[1/8] Search: {query}")
+            prog(0.05, desc="Searching...")
+            res = search_fancaps(query, anime, movies, tv, log_fn=L)
+            if not res:
+                return None, "\n".join(log) + "\n\nSearch blocked! Use direct fancaps URL."
+            item = next((items[0] for items in res.values() if items), None)
+            if not item: return None, "No results!"
+
+        show_name = item[0]
+        if not char: char = sanitize(show_name)
+        t1 = time.time(); L(f"  Found: {show_name} ({t1-t0:.0f}s)"); prog(0.1, desc="Downloading...")
+
+        # [2/8] Scrape
+        scrape(item[0], item[1], dirs['scrapped'], max_img, progress if not cli_mode else None, log_fn=L)
+        imgs = get_imgs(dirs['scrapped'])
+        t2 = time.time(); L(f"[2/8] Downloaded: {len(imgs)} ({t2-t1:.0f}s)")
+        if not imgs: return None, "No images downloaded!"
+
+        # [3/8] Dedup
+        prog(0.3, desc="Dedup...")
+        imgs = [p for p in imgs if valid_img(p)]
+        kept, rm = dedup(imgs)
+        for p in kept: shutil.copy(p, os.path.join(dirs['filtered'], os.path.basename(p)))
+        t3 = time.time(); L(f"[3/8] Dedup: {len(kept)} kept, -{len(rm)} ({t3-t2:.0f}s)")
+
+        # [4/8] Detect faces
+        prog(0.4, desc="Faces...")
+        n = detect_faces(get_imgs(dirs['filtered']), dirs['faces'])
+        t4 = time.time(); L(f"[4/8] Faces: {n} ({t4-t3:.0f}s)")
+        if n == 0: return None, "No faces detected!"
+
+        # [5/8] Process examples
+        prog(0.5, desc="Examples...")
+        ex_paths = [p for p in (examples or []) if p and os.path.exists(p)]
+        if not ex_paths: ex_paths = [p for p in EXAMPLES if os.path.exists(p)]
+        if not ex_paths: return None, "No example images!"
+        n_ex = detect_faces(ex_paths, dirs['ex_faces'])
+        t5 = time.time(); L(f"[5/8] Examples: {len(ex_paths)} imgs -> {n_ex} faces ({t5-t4:.0f}s)")
+        if n_ex == 0: return None, "No faces in examples!"
+
+        # [6/8] Match
+        prog(0.6, desc="Matching...")
+        f_emb, f_valid = face_emb(get_imgs(dirs['faces']))
+        e_emb, _ = face_emb(get_imgs(dirs['ex_faces']))
+        dists = pairwise_distances(f_emb, e_emb, metric='euclidean').min(axis=1)
+        similar_idx = np.where(dists < thresh)[0]
+        similar = [f_valid[i] for i in similar_idx]
+        similar_dists = dists[similar_idx]
+        t6 = time.time()
+        L(f"[6/8] Matches: {len(similar)} (thresh={thresh}) ({t6-t5:.0f}s)")
+        if len(similar_dists) > 0:
+            L(f"  Distances: min={similar_dists.min():.1f}, max={similar_dists.max():.1f}, mean={similar_dists.mean():.1f}")
+        if not similar: return None, f"No matches! Try threshold > {thresh}"
+
+        # [7/8] Get originals
+        prog(0.7, desc="Collect...")
+        origs = set()
+        orig_to_dist = {}
+        for i, fp in enumerate(similar):
+            parts = os.path.basename(fp).rsplit('-', 2)
+            base = parts[0] if len(parts) >= 3 else Path(fp).stem
+            for ext in EXTS:
+                op = os.path.join(dirs['filtered'], base + ext)
+                if os.path.exists(op):
+                    origs.add(op)
+                    orig_to_dist[os.path.basename(op)] = similar_dists[i]
+                    break
+        res_dir = os.path.join(work, f"results_{sanitize(char)}")
+        os.makedirs(res_dir, exist_ok=True)
+        for p in origs: shutil.copy(p, os.path.join(res_dir, os.path.basename(p)))
+        t7 = time.time(); L(f"[7/8] Collected: {len(origs)} ({t7-t6:.0f}s)")
+
+        # [8/8] Tag
+        prog(0.8, desc="Tagging...")
+        char_tag = char if char != sanitize(show_name) else ""
+        for p in get_imgs(res_dir):
+            tags = tag(p, act_tag, char_tag)
+            with open(os.path.splitext(p)[0] + ".txt", 'w') as f: f.write(", ".join(tags))
+        t8 = time.time(); L(f"[8/8] Tagged: {len(origs)} ({t8-t7:.0f}s)")
+
+        # Log each image with distance
+        L(f"\nResults (distance to ref):")
+        for name, d in sorted(orig_to_dist.items(), key=lambda x: x[1]):
+            L(f"  {name}: {d:.1f}")
+
+        # Zip
+        prog(0.95, desc="Zipping...")
+        zp = os.path.join(work, f"{sanitize(char)}_dataset.zip")
+        with zipfile.ZipFile(zp, 'w', zipfile.ZIP_DEFLATED) as z:
+            for p in get_imgs(res_dir) + [os.path.splitext(p)[0]+".txt" for p in get_imgs(res_dir)]:
+                if os.path.exists(p): z.write(p, os.path.basename(p))
+        # Copy ZIP to persistent temp location (Gradio needs file to exist after return)
+        final_zip = tempfile.NamedTemporaryFile(delete=False, suffix=".zip", prefix=f"{sanitize(char)}_").name
+        shutil.copy(zp, final_zip)
+        L(f"\nDone! {len(origs)} images, total {t8-t0:.0f}s"); prog(1.0, desc="Complete!")
+        return final_zip, "\n".join(log)
+    except Exception as e:
+        import traceback
+        return None, "\n".join(log) + f"\n\nERROR: {e}\n{traceback.format_exc()}"
+    finally:
+        # Clean up work directory (ZIP already copied out)
+        if os.path.exists(work):
+            shutil.rmtree(work, ignore_errors=True)
+
+# =============================================================================
+# UI
+# =============================================================================
+css = """
+.gradio-container { padding-top: 10px !important; }
+.compact-group { margin-bottom: 8px !important; }
+"""
+
+with gr.Blocks(title="SD Dataset Automaker: Fancaps → YOLO → Siamese → WD Tag → ZIP") as demo:
+    gr.Markdown("### SD Dataset Automaker: Fancaps → YOLO → Siamese → WD → ZIP")
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            # Compact input group
+            with gr.Group():
+                with gr.Row():
+                    query = gr.Textbox(
+                        label="Anime / Movie / Fancaps URL",
+                        placeholder="'Cowboy Bebop' or paste URL",
+                        scale=3
+                    )
+                    char = gr.Textbox(label="Character (optional, appends to tags)", placeholder="spike_spiegel", scale=2)
+
+            with gr.Row():
+                ref_imgs = gr.Gallery(
+                    label="Reference Face Image(s)",
+                    columns=4,
+                    height=100,
+                    interactive=True,
+                    object_fit="scale-down",
+                    scale=3,
+                )
+                run_btn = gr.Button("Generate Dataset", variant="primary", size="lg", scale=1)
+
+            # Hidden File input for MCP compatibility (Gallery $ref schema bug persists in Gradio 6.0.1)
+            ref_files = gr.File(
+                label="Reference Images (MCP)",
+                file_count="multiple",
+                file_types=["image"],
+                visible=False,
+            )
+
+            # gr.Examples + gr.Gallery works in Gradio 5.46.0+ (PR #11787)
+            gr.Examples(
+                examples=[
+                    ["https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop", "spike_spiegel", EXAMPLES],
+                ],
+                inputs=[query, char, ref_imgs],
+                label="Example (click to load)",
+            )
+
+            # Advanced settings in accordion
+            with gr.Accordion("Advanced Settings", open=False):
+                with gr.Row():
+                    max_img = gr.Slider(50, 500, 200, step=50, label="Max Downloads (frames to scrape)")
+                    thresh = gr.Slider(20, 60, 32, step=1, label="Face Similarity (lower=stricter)")
+                with gr.Row():
+                    act_tag = gr.Textbox(label="Trigger Word (prepends to captions)", placeholder="e.g. sks_style", scale=2)
+                    anime_cb = gr.Checkbox(label="Anime", value=True, scale=1)
+                    movies_cb = gr.Checkbox(label="Movies", scale=1)
+                    tv_cb = gr.Checkbox(label="TV", scale=1)
+
+        with gr.Column(scale=1):
+            out_file = gr.File(label="Download ZIP", interactive=False)
+            with gr.Accordion("Log", open=True):
+                out_log = gr.Textbox(label="", lines=12, max_lines=50, show_label=False, autoscroll=False)
+            gr.Markdown("*CPU: ~5-10 min/run*")
+
+    def process(q, c, imgs, files, mi, th, at, an, mo, tv, prog=gr.Progress()):
+        if not q:
+            gr.Warning("Enter anime name or URL")
+            return None, ""
+        # Collect paths from Gallery (imgs) or File input (files)
+        paths = []
+        for item in (imgs or []):
+            p = item[0] if isinstance(item, (list, tuple)) else item
+            if p and os.path.exists(p): paths.append(p)
+        if not paths and files:
+            for f in (files if isinstance(files, list) else [files]):
+                fp = f.name if hasattr(f, 'name') else str(f)
+                if fp and os.path.exists(fp): paths.append(fp)
+        if not paths:
+            gr.Warning("Upload reference images or click 'Load Example'")
+            return None, ""
+        if 'fancaps.net' in q:
+            gr.Info("Direct URL detected")
+        else:
+            gr.Info(f"Searching: {q}")
+        zp, log = run(q, c, paths, mi, th, at, an, mo, tv, prog)
+        if zp:
+            gr.Info("Done!")
+        return zp, log
+
+    run_btn.click(process, [query, char, ref_imgs, ref_files, max_img, thresh, act_tag, anime_cb, movies_cb, tv_cb], [out_file, out_log])
+
+def run_cli():
+    """CLI mode with cloudscraper for Cloudflare bypass"""
+    # Use cloudscraper for CLI (bypasses Cloudflare on local/residential IPs)
+    init_session(use_cloudscraper=True)
+
+    parser = argparse.ArgumentParser(description="SD Dataset Automaker - Anime character dataset generator")
+    parser.add_argument("--title", "-t", required=True, help="Anime name or fancaps.net URL")
+    parser.add_argument("--image", "-i", nargs="+", required=True, help="Reference face images (1-5)")
+    parser.add_argument("--char", "-c", default="", help="Character name (optional, appends to tags)")
+    parser.add_argument("--max", "-m", type=int, default=200, help="Max frames to scrape (default: 200)")
+    parser.add_argument("--thresh", type=float, default=32.0, help="Face similarity threshold, lower=stricter (default: 32)")
+    parser.add_argument("--tag", default="", help="Trigger word to prepend to captions")
+    parser.add_argument("--anime", action="store_true", default=True, help="Search anime (default)")
+    parser.add_argument("--movies", action="store_true", help="Search movies")
+    parser.add_argument("--tv", action="store_true", help="Search TV")
+    parser.add_argument("--output", "-o", default=".", help="Output directory (default: current)")
+    args = parser.parse_args()
+
+    # Validate images
+    ref_imgs = [p for p in args.image if os.path.exists(p)]
+    if not ref_imgs:
+        print(f"ERROR: No valid reference images found: {args.image}")
+        sys.exit(1)
+
+    print(f"SD Dataset Automaker - CLI Mode")
+    print(f"  Title: {args.title}")
+    print(f"  Refs:  {len(ref_imgs)} images")
+    print(f"  Char:  {args.char or '(auto from title)'}")
+    print()
+
+    zp, log = run(
+        query=args.title,
+        char=args.char,
+        examples=ref_imgs,
+        max_img=args.max,
+        thresh=args.thresh,
+        act_tag=args.tag,
+        anime=args.anime,
+        movies=args.movies,
+        tv=args.tv,
+        cli_mode=True
+    )
+
+    if zp:
+        # Copy to output dir
+        out_path = os.path.join(args.output, os.path.basename(zp))
+        shutil.copy(zp, out_path)
+        print(f"\nSaved: {out_path}")
+    else:
+        print(f"\nFailed!")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    # CLI mode if args provided, else Gradio UI
+    if len(sys.argv) > 1:
+        run_cli()
+    else:
+        # Gradio UI mode
+        allowed_dir = os.path.dirname(os.path.abspath(__file__))
+        demo.launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            mcp_server=True,
+            show_error=True,
+            allowed_paths=[allowed_dir],
+            css=css,
+        )

gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+from_url_spike_spiegel1.jpg filter=lfs diff=lfs merge=lfs -text

requirements.txt

@@ -0,0 +1,14 @@
+gradio[mcp]
+huggingface_hub
+torch>=2.0.0
+torchvision>=0.15.0
+numpy>=1.24.0
+Pillow>=10.0.0
+scikit-learn>=1.3.0
+pandas>=2.1.0
+onnxruntime>=1.18.0
+requests>=2.31.0
+beautifulsoup4>=4.12.0
+ultralytics>=8.2.0
+seaborn>=0.13.0
+tqdm>=4.66.0

similarity.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:225a605835f1b65b0fb82d551c316d2b5605fd7e85fd36682a6905ac5a3db9ad
+size 21465851

yolov5s_anime.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:237e6fa848b1a2e276465c17cca933f27f498ed701ab12e7c06d1ef8959c62bf
+size 14756994