Dinamush Cursor commited on
Commit ·
fdb87aa
1
Parent(s): 6aacf6e
chore: remove personal paths and album names from tooling
Browse files
backend/app/realism_sources.py
CHANGED
|
@@ -133,12 +133,19 @@ def fetch_randomuser_people_photos(limit: int) -> list[RealismSample]:
|
|
| 133 |
|
| 134 |
|
| 135 |
def fetch_local_people_photos(limit: int) -> list[RealismSample]:
|
| 136 |
-
"""Fallback: local
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
exts = {".jpg", ".jpeg", ".png", ".webp", ".jfif"}
|
| 143 |
found: list[Path] = []
|
| 144 |
for root in roots:
|
|
@@ -166,7 +173,6 @@ def fetch_local_people_photos(limit: int) -> list[RealismSample]:
|
|
| 166 |
)
|
| 167 |
return samples
|
| 168 |
|
| 169 |
-
|
| 170 |
def fetch_commons_people_photos(limit: int) -> list[RealismSample]:
|
| 171 |
"""Pull real people photographs from Wikimedia Commons search."""
|
| 172 |
if limit < 1:
|
|
|
|
| 133 |
|
| 134 |
|
| 135 |
def fetch_local_people_photos(limit: int) -> list[RealismSample]:
|
| 136 |
+
"""Fallback: local photo folders when remotes fail.
|
| 137 |
+
|
| 138 |
+
Set THR3SHR_LOCAL_PHOTO_DIRS to a pathsep/semicolon-separated list of dirs.
|
| 139 |
+
Defaults to ~/Pictures only (no personal album names).
|
| 140 |
+
"""
|
| 141 |
+
import os
|
| 142 |
+
|
| 143 |
+
raw = os.environ.get("THR3SHR_LOCAL_PHOTO_DIRS", "").strip()
|
| 144 |
+
if raw:
|
| 145 |
+
sep = ";" if ";" in raw else os.pathsep
|
| 146 |
+
roots = [Path(p.strip()) for p in raw.split(sep) if p.strip()]
|
| 147 |
+
else:
|
| 148 |
+
roots = [Path.home() / "Pictures"]
|
| 149 |
exts = {".jpg", ".jpeg", ".png", ".webp", ".jfif"}
|
| 150 |
found: list[Path] = []
|
| 151 |
for root in roots:
|
|
|
|
| 173 |
)
|
| 174 |
return samples
|
| 175 |
|
|
|
|
| 176 |
def fetch_commons_people_photos(limit: int) -> list[RealismSample]:
|
| 177 |
"""Pull real people photographs from Wikimedia Commons search."""
|
| 178 |
if limit < 1:
|
backend/scripts/bench_batch_throughput.py
CHANGED
|
@@ -2,12 +2,12 @@
|
|
| 2 |
"""Compare WD single vs batched ORT throughput on local images.
|
| 3 |
|
| 4 |
Usage (from backend/):
|
| 5 |
-
../.venv/Scripts/python.exe scripts/bench_batch_throughput.py
|
| 6 |
-
../.venv/Scripts/python.exe scripts/bench_batch_throughput.py --root "E:\\path" --n 24
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
import argparse
|
|
|
|
| 11 |
import time
|
| 12 |
from pathlib import Path
|
| 13 |
|
|
@@ -57,7 +57,8 @@ def main() -> int:
|
|
| 57 |
parser = argparse.ArgumentParser()
|
| 58 |
parser.add_argument(
|
| 59 |
"--root",
|
| 60 |
-
default=
|
|
|
|
| 61 |
)
|
| 62 |
parser.add_argument("--n", type=int, default=24)
|
| 63 |
parser.add_argument("--model", default="wd_swinv2_v3")
|
|
@@ -65,6 +66,9 @@ def main() -> int:
|
|
| 65 |
parser.add_argument("--batches", default="1,4,8")
|
| 66 |
args = parser.parse_args()
|
| 67 |
|
|
|
|
|
|
|
|
|
|
| 68 |
root = Path(args.root)
|
| 69 |
images = _collect_images(root, args.n)
|
| 70 |
if len(images) < 4:
|
|
|
|
| 2 |
"""Compare WD single vs batched ORT throughput on local images.
|
| 3 |
|
| 4 |
Usage (from backend/):
|
| 5 |
+
../.venv/Scripts/python.exe scripts/bench_batch_throughput.py --root /path/to/images --n 24
|
|
|
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import argparse
|
| 10 |
+
import os
|
| 11 |
import time
|
| 12 |
from pathlib import Path
|
| 13 |
|
|
|
|
| 57 |
parser = argparse.ArgumentParser()
|
| 58 |
parser.add_argument(
|
| 59 |
"--root",
|
| 60 |
+
default=os.environ.get("THR3SHR_BENCH_ROOT", ""),
|
| 61 |
+
help="Image root (or set THR3SHR_BENCH_ROOT)",
|
| 62 |
)
|
| 63 |
parser.add_argument("--n", type=int, default=24)
|
| 64 |
parser.add_argument("--model", default="wd_swinv2_v3")
|
|
|
|
| 66 |
parser.add_argument("--batches", default="1,4,8")
|
| 67 |
args = parser.parse_args()
|
| 68 |
|
| 69 |
+
if not str(args.root).strip():
|
| 70 |
+
raise SystemExit("Pass --root /path/to/images or set THR3SHR_BENCH_ROOT")
|
| 71 |
+
|
| 72 |
root = Path(args.root)
|
| 73 |
images = _collect_images(root, args.n)
|
| 74 |
if len(images) < 4:
|
backend/scripts/debug_realism_probe.py
CHANGED
|
@@ -13,6 +13,7 @@ from __future__ import annotations
|
|
| 13 |
|
| 14 |
import argparse
|
| 15 |
import json
|
|
|
|
| 16 |
import sqlite3
|
| 17 |
import statistics
|
| 18 |
import sys
|
|
@@ -80,20 +81,17 @@ PRIMARY_EVIDENCE_TAGS = (
|
|
| 80 |
"3d_background",
|
| 81 |
)
|
| 82 |
|
| 83 |
-
#
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
]
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
Path(r"/examples/anime/d"),
|
| 95 |
-
Path(r"/examples/anime/e"),
|
| 96 |
-
Path(r"~/Pictures"),
|
| 97 |
Path(__file__).resolve().parents[2] / "sample_data" / "sfw_safebooru",
|
| 98 |
]
|
| 99 |
|
|
@@ -238,19 +236,25 @@ def collect_labeled_sets(
|
|
| 238 |
if found:
|
| 239 |
photo_paths.extend(found)
|
| 240 |
notes.append(f"auto photo dir {d} (+{len(found)})")
|
| 241 |
-
# WhatsApp /
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
|
| 255 |
anime_paths: list[Path] = []
|
| 256 |
if anime_dir:
|
|
|
|
| 13 |
|
| 14 |
import argparse
|
| 15 |
import json
|
| 16 |
+
import os
|
| 17 |
import sqlite3
|
| 18 |
import statistics
|
| 19 |
import sys
|
|
|
|
| 81 |
"3d_background",
|
| 82 |
)
|
| 83 |
|
| 84 |
+
# Optional local dirs via env (colon/semicolon-separated). Empty = Wikimedia fallbacks.
|
| 85 |
+
def _env_dirs(name: str) -> list[Path]:
|
| 86 |
+
raw = os.environ.get(name, "").strip()
|
| 87 |
+
if not raw:
|
| 88 |
+
return []
|
| 89 |
+
sep = ";" if ";" in raw else os.pathsep
|
| 90 |
+
return [Path(p.strip()) for p in raw.split(sep) if p.strip()]
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
DEFAULT_PHOTO_DIRS = _env_dirs("THR3SHR_PROBE_PHOTO_DIRS")
|
| 94 |
+
DEFAULT_ANIME_DIRS = _env_dirs("THR3SHR_PROBE_ANIME_DIRS") + [
|
|
|
|
|
|
|
|
|
|
| 95 |
Path(__file__).resolve().parents[2] / "sample_data" / "sfw_safebooru",
|
| 96 |
]
|
| 97 |
|
|
|
|
| 236 |
if found:
|
| 237 |
photo_paths.extend(found)
|
| 238 |
notes.append(f"auto photo dir {d} (+{len(found)})")
|
| 239 |
+
# Optional: scan Pictures for WhatsApp / camera dumps when enabled
|
| 240 |
+
if os.environ.get("THR3SHR_PROBE_SCAN_PICTURES", "").strip().lower() in {
|
| 241 |
+
"1",
|
| 242 |
+
"true",
|
| 243 |
+
"yes",
|
| 244 |
+
"on",
|
| 245 |
+
}:
|
| 246 |
+
pics_root = Path.home() / "Pictures"
|
| 247 |
+
if pics_root.exists():
|
| 248 |
+
extras = []
|
| 249 |
+
for p in pics_root.iterdir():
|
| 250 |
+
if not p.is_file() or p.suffix.lower() not in IMAGE_EXTS:
|
| 251 |
+
continue
|
| 252 |
+
name = p.name.lower()
|
| 253 |
+
if "whatsapp" in name or name.startswith("img_"):
|
| 254 |
+
extras.append(p)
|
| 255 |
+
if extras:
|
| 256 |
+
photo_paths.extend(extras)
|
| 257 |
+
notes.append(f"auto Pictures root camera-like (+{len(extras)})")
|
| 258 |
|
| 259 |
anime_paths: list[Path] = []
|
| 260 |
if anime_dir:
|
backend/scripts/run_realism_debug_eval.py
CHANGED
|
@@ -7,6 +7,7 @@ from __future__ import annotations
|
|
| 7 |
|
| 8 |
import json
|
| 9 |
import logging
|
|
|
|
| 10 |
import sys
|
| 11 |
from datetime import datetime, timezone
|
| 12 |
from pathlib import Path
|
|
@@ -35,11 +36,13 @@ def _settings_from_db() -> AppSettings:
|
|
| 35 |
|
| 36 |
|
| 37 |
def _probe_local_videos(settings: AppSettings, model: str, limit: int = 8) -> dict:
|
| 38 |
-
"""Score local
|
| 39 |
-
roots = [
|
| 40 |
-
|
| 41 |
-
Path(
|
| 42 |
-
|
|
|
|
|
|
|
| 43 |
videos: list[Path] = []
|
| 44 |
for root in roots:
|
| 45 |
if root is None or not root.is_dir():
|
|
|
|
| 7 |
|
| 8 |
import json
|
| 9 |
import logging
|
| 10 |
+
import os
|
| 11 |
import sys
|
| 12 |
from datetime import datetime, timezone
|
| 13 |
from pathlib import Path
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
def _probe_local_videos(settings: AppSettings, model: str, limit: int = 8) -> dict:
|
| 39 |
+
"""Score local videos with multi-frame pooling if present."""
|
| 40 |
+
roots = []
|
| 41 |
+
if settings.root_repo:
|
| 42 |
+
roots.append(Path(settings.root_repo).expanduser())
|
| 43 |
+
extra = os.environ.get("THR3SHR_VIDEO_PROBE_ROOT", "").strip()
|
| 44 |
+
if extra:
|
| 45 |
+
roots.append(Path(extra).expanduser())
|
| 46 |
videos: list[Path] = []
|
| 47 |
for root in roots:
|
| 48 |
if root is None or not root.is_dir():
|
backend/scripts/verify_all_taggers.py
CHANGED
|
@@ -2,11 +2,13 @@
|
|
| 2 |
End-to-end verification of all tagger models against a small image folder.
|
| 3 |
|
| 4 |
Usage (from backend/):
|
|
|
|
| 5 |
../.venv/Scripts/python.exe scripts/verify_all_taggers.py
|
| 6 |
"""
|
| 7 |
from __future__ import annotations
|
| 8 |
|
| 9 |
import json
|
|
|
|
| 10 |
import sys
|
| 11 |
import time
|
| 12 |
import urllib.error
|
|
@@ -14,8 +16,8 @@ import urllib.request
|
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
API = "http://127.0.0.1:8000/api"
|
| 17 |
-
ROOT = Path(
|
| 18 |
-
CATS = ROOT / "cats"
|
| 19 |
# Keep selected tags broad enough that anime stills usually hit something.
|
| 20 |
SELECTED = ["1girl", "solo", "long_hair", "blush", "smile"]
|
| 21 |
MODELS = ["ml_danbooru", "wd_swinv2_v3", "wd_eva02_large"]
|
|
@@ -59,8 +61,12 @@ def wait_run(run_id: int, timeout_s: float = 1800.0) -> dict:
|
|
| 59 |
|
| 60 |
|
| 61 |
def main() -> int:
|
| 62 |
-
if not ROOT.is_dir():
|
| 63 |
-
print(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
return 2
|
| 65 |
CATS.mkdir(parents=True, exist_ok=True)
|
| 66 |
|
|
@@ -180,21 +186,30 @@ def main() -> int:
|
|
| 180 |
if not ok:
|
| 181 |
print(f"FAIL model={model}", file=sys.stderr)
|
| 182 |
|
| 183 |
-
#
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
print("\n=== SUMMARY ===")
|
| 200 |
all_ok = True
|
|
|
|
| 2 |
End-to-end verification of all tagger models against a small image folder.
|
| 3 |
|
| 4 |
Usage (from backend/):
|
| 5 |
+
set THR3SHR_PROBE_ROOT=C:\\path\\to\\probe_images
|
| 6 |
../.venv/Scripts/python.exe scripts/verify_all_taggers.py
|
| 7 |
"""
|
| 8 |
from __future__ import annotations
|
| 9 |
|
| 10 |
import json
|
| 11 |
+
import os
|
| 12 |
import sys
|
| 13 |
import time
|
| 14 |
import urllib.error
|
|
|
|
| 16 |
from pathlib import Path
|
| 17 |
|
| 18 |
API = "http://127.0.0.1:8000/api"
|
| 19 |
+
ROOT = Path(os.environ.get("THR3SHR_PROBE_ROOT", "")).expanduser()
|
| 20 |
+
CATS = ROOT / "cats" if ROOT.parts else Path("cats")
|
| 21 |
# Keep selected tags broad enough that anime stills usually hit something.
|
| 22 |
SELECTED = ["1girl", "solo", "long_hair", "blush", "smile"]
|
| 23 |
MODELS = ["ml_danbooru", "wd_swinv2_v3", "wd_eva02_large"]
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
def main() -> int:
|
| 64 |
+
if not ROOT.parts or not ROOT.is_dir():
|
| 65 |
+
print(
|
| 66 |
+
"ERROR: set THR3SHR_PROBE_ROOT to a folder of probe images "
|
| 67 |
+
"(with optional cats/ subcategory root).",
|
| 68 |
+
file=sys.stderr,
|
| 69 |
+
)
|
| 70 |
return 2
|
| 71 |
CATS.mkdir(parents=True, exist_ok=True)
|
| 72 |
|
|
|
|
| 186 |
if not ok:
|
| 187 |
print(f"FAIL model={model}", file=sys.stderr)
|
| 188 |
|
| 189 |
+
# Optionally restore durable library paths for the user UI (tags preserved).
|
| 190 |
+
restore_root = os.environ.get("THR3SHR_RESTORE_ROOT_REPO", "").strip()
|
| 191 |
+
restore_cats = os.environ.get("THR3SHR_RESTORE_CATEGORIES_ROOT", "").strip()
|
| 192 |
+
if restore_root and restore_cats:
|
| 193 |
+
user_settings = req("GET", "/settings")
|
| 194 |
+
user_settings.update(
|
| 195 |
+
{
|
| 196 |
+
"root_repo": restore_root,
|
| 197 |
+
"categories_root": restore_cats,
|
| 198 |
+
"selected_tags": [],
|
| 199 |
+
"tagger_model": "wd_swinv2_v3",
|
| 200 |
+
"confidence_threshold": 0.6,
|
| 201 |
+
"wd_general_threshold": 0.35,
|
| 202 |
+
"max_inference_workers": 2,
|
| 203 |
+
}
|
| 204 |
+
)
|
| 205 |
+
req("PUT", "/settings", user_settings)
|
| 206 |
+
print("\nRestored UI settings from THR3SHR_RESTORE_* env vars", flush=True)
|
| 207 |
+
else:
|
| 208 |
+
print(
|
| 209 |
+
"\nSkipped UI path restore (set THR3SHR_RESTORE_ROOT_REPO and "
|
| 210 |
+
"THR3SHR_RESTORE_CATEGORIES_ROOT to restore).",
|
| 211 |
+
flush=True,
|
| 212 |
+
)
|
| 213 |
|
| 214 |
print("\n=== SUMMARY ===")
|
| 215 |
all_ok = True
|