Spaces:
Sleeping
Sleeping
Sync from kink_cli (Docker Space)
Browse files- .gitignore +1 -0
- api.py +26 -11
- backend/hf_bootstrap.py +45 -0
- backend/media_remote.py +141 -0
- frontend/components.js +33 -6
- frontend/dist/assets/index-DoKOnBon.js +0 -0
- frontend/dist/index.html +1 -1
- frontend/my-plays.js +30 -4
- scripts/sync_hf_space_b2_media.py +208 -0
- scripts/verify_hf_stack.py +11 -1
- tests/test_api_media_fallback.py +50 -5
- tests/test_hf_bootstrap.py +35 -0
.gitignore
CHANGED
|
@@ -13,6 +13,7 @@ data/store.db-shm
|
|
| 13 |
data/store_slim.db
|
| 14 |
data/store_slim.db-wal
|
| 15 |
data/store_slim.db-shm
|
|
|
|
| 16 |
data/recsys/
|
| 17 |
|
| 18 |
# Python
|
|
|
|
| 13 |
data/store_slim.db
|
| 14 |
data/store_slim.db-wal
|
| 15 |
data/store_slim.db-shm
|
| 16 |
+
data/cached_assets*.tar
|
| 17 |
data/recsys/
|
| 18 |
|
| 19 |
# Python
|
api.py
CHANGED
|
@@ -11,12 +11,13 @@ from typing import Any
|
|
| 11 |
|
| 12 |
from fastapi import FastAPI, Header, HTTPException, Query, Request
|
| 13 |
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
-
from fastapi.responses import FileResponse, HTMLResponse
|
| 15 |
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
| 16 |
from sse_starlette.sse import EventSourceResponse
|
| 17 |
|
| 18 |
from backend import Backend, VALID_RATINGS
|
| 19 |
-
from backend.hf_bootstrap import ensure_store_db
|
|
|
|
| 20 |
|
| 21 |
_app_dir = Path(__file__).resolve().parent
|
| 22 |
_data_dir = _app_dir / "data"
|
|
@@ -54,18 +55,18 @@ def _require_frontend_build() -> None:
|
|
| 54 |
def _hf_space_published_image() -> bool:
|
| 55 |
"""True in the Hugging Face Space Docker image (``Dockerfile`` sets ``KINK_HF_SPACE_IMAGE=1``).
|
| 56 |
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
"""
|
| 61 |
return _env_truthy("KINK_HF_SPACE_IMAGE")
|
| 62 |
|
| 63 |
|
| 64 |
def _media_strict_missing() -> bool:
|
| 65 |
"""If true, missing ``/media/...`` bytes return 404 instead of a same-origin placeholder image."""
|
| 66 |
-
if
|
| 67 |
-
return
|
| 68 |
-
return _env_truthy("
|
| 69 |
|
| 70 |
|
| 71 |
def _media_fallback_path() -> Path | None:
|
|
@@ -85,13 +86,15 @@ def _media_fallback_path() -> Path | None:
|
|
| 85 |
|
| 86 |
def _media_health_payload() -> dict[str, Any]:
|
| 87 |
fb = _media_fallback_path()
|
| 88 |
-
|
| 89 |
"hf_space_image": _hf_space_published_image(),
|
| 90 |
"strict_env": _env_truthy("KINK_MEDIA_STRICT"),
|
| 91 |
"strict_missing_assets": _media_strict_missing(),
|
| 92 |
"fallback_ready": fb is not None,
|
| 93 |
"fallback_path": str(fb) if fb is not None else None,
|
| 94 |
}
|
|
|
|
|
|
|
| 95 |
|
| 96 |
_backend_impl: Backend | None = None
|
| 97 |
_backend_lock = threading.Lock()
|
|
@@ -101,6 +104,10 @@ def _env_truthy(name: str) -> bool:
|
|
| 101 |
return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on")
|
| 102 |
|
| 103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
def _store_path_is_ephemeral(path: Path) -> bool:
|
| 105 |
resolved = path.resolve()
|
| 106 |
prefixes = ("/tmp", "/var/tmp", "/dev/shm")
|
|
@@ -134,6 +141,7 @@ def _get_backend() -> Backend:
|
|
| 134 |
with _backend_lock:
|
| 135 |
if _backend_impl is not None:
|
| 136 |
return _backend_impl
|
|
|
|
| 137 |
path = ensure_store_db(_default_store)
|
| 138 |
_warn_or_fail_ephemeral_store(path)
|
| 139 |
b = Backend(path)
|
|
@@ -399,13 +407,20 @@ def favicon() -> FileResponse:
|
|
| 399 |
raise HTTPException(status_code=404, detail="Not found")
|
| 400 |
|
| 401 |
|
| 402 |
-
@app.get("/media/{bucket}/{filename:path}")
|
| 403 |
-
def media(bucket: str, filename: str) ->
|
| 404 |
target = (MEDIA_ROOT / bucket / filename).resolve()
|
| 405 |
if MEDIA_ROOT.resolve() not in target.parents:
|
| 406 |
raise HTTPException(status_code=404, detail="Unknown asset")
|
| 407 |
if target.is_file():
|
| 408 |
return FileResponse(target, headers={"Cache-Control": _MEDIA_CACHE})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
if not _media_strict_missing():
|
| 410 |
fb = _media_fallback_path()
|
| 411 |
if fb is not None:
|
|
|
|
| 11 |
|
| 12 |
from fastapi import FastAPI, Header, HTTPException, Query, Request
|
| 13 |
from fastapi.middleware.cors import CORSMiddleware
|
| 14 |
+
from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
|
| 15 |
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
| 16 |
from sse_starlette.sse import EventSourceResponse
|
| 17 |
|
| 18 |
from backend import Backend, VALID_RATINGS
|
| 19 |
+
from backend.hf_bootstrap import ensure_media_cache, ensure_store_db
|
| 20 |
+
from backend.media_remote import remote_media_health_payload, remote_media_url
|
| 21 |
|
| 22 |
_app_dir = Path(__file__).resolve().parent
|
| 23 |
_data_dir = _app_dir / "data"
|
|
|
|
| 55 |
def _hf_space_published_image() -> bool:
|
| 56 |
"""True in the Hugging Face Space Docker image (``Dockerfile`` sets ``KINK_HF_SPACE_IMAGE=1``).
|
| 57 |
|
| 58 |
+
Seed deployments can serve bundled JPEG fallbacks for missing catalog tiles. Full-catalog
|
| 59 |
+
deployments must expose missing bytes as 404s so the UI can use a neutral placeholder instead of
|
| 60 |
+
repeating an unrelated demo image.
|
| 61 |
"""
|
| 62 |
return _env_truthy("KINK_HF_SPACE_IMAGE")
|
| 63 |
|
| 64 |
|
| 65 |
def _media_strict_missing() -> bool:
|
| 66 |
"""If true, missing ``/media/...`` bytes return 404 instead of a same-origin placeholder image."""
|
| 67 |
+
if _env_is_set("KINK_MEDIA_STRICT"):
|
| 68 |
+
return _env_truthy("KINK_MEDIA_STRICT")
|
| 69 |
+
return _env_truthy("KINK_HF_REQUIRE_FULL_CATALOG")
|
| 70 |
|
| 71 |
|
| 72 |
def _media_fallback_path() -> Path | None:
|
|
|
|
| 86 |
|
| 87 |
def _media_health_payload() -> dict[str, Any]:
|
| 88 |
fb = _media_fallback_path()
|
| 89 |
+
payload = {
|
| 90 |
"hf_space_image": _hf_space_published_image(),
|
| 91 |
"strict_env": _env_truthy("KINK_MEDIA_STRICT"),
|
| 92 |
"strict_missing_assets": _media_strict_missing(),
|
| 93 |
"fallback_ready": fb is not None,
|
| 94 |
"fallback_path": str(fb) if fb is not None else None,
|
| 95 |
}
|
| 96 |
+
payload.update(remote_media_health_payload())
|
| 97 |
+
return payload
|
| 98 |
|
| 99 |
_backend_impl: Backend | None = None
|
| 100 |
_backend_lock = threading.Lock()
|
|
|
|
| 104 |
return os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on")
|
| 105 |
|
| 106 |
|
| 107 |
+
def _env_is_set(name: str) -> bool:
|
| 108 |
+
return os.environ.get(name, "").strip() != ""
|
| 109 |
+
|
| 110 |
+
|
| 111 |
def _store_path_is_ephemeral(path: Path) -> bool:
|
| 112 |
resolved = path.resolve()
|
| 113 |
prefixes = ("/tmp", "/var/tmp", "/dev/shm")
|
|
|
|
| 141 |
with _backend_lock:
|
| 142 |
if _backend_impl is not None:
|
| 143 |
return _backend_impl
|
| 144 |
+
ensure_media_cache(MEDIA_ROOT)
|
| 145 |
path = ensure_store_db(_default_store)
|
| 146 |
_warn_or_fail_ephemeral_store(path)
|
| 147 |
b = Backend(path)
|
|
|
|
| 407 |
raise HTTPException(status_code=404, detail="Not found")
|
| 408 |
|
| 409 |
|
| 410 |
+
@app.get("/media/{bucket}/{filename:path}", response_model=None)
|
| 411 |
+
def media(bucket: str, filename: str) -> Any:
|
| 412 |
target = (MEDIA_ROOT / bucket / filename).resolve()
|
| 413 |
if MEDIA_ROOT.resolve() not in target.parents:
|
| 414 |
raise HTTPException(status_code=404, detail="Unknown asset")
|
| 415 |
if target.is_file():
|
| 416 |
return FileResponse(target, headers={"Cache-Control": _MEDIA_CACHE})
|
| 417 |
+
remote_url = remote_media_url(f"{bucket}/{filename}")
|
| 418 |
+
if remote_url:
|
| 419 |
+
return RedirectResponse(
|
| 420 |
+
remote_url,
|
| 421 |
+
status_code=307,
|
| 422 |
+
headers={"Cache-Control": "private, max-age=300"},
|
| 423 |
+
)
|
| 424 |
if not _media_strict_missing():
|
| 425 |
fb = _media_fallback_path()
|
| 426 |
if fb is not None:
|
backend/hf_bootstrap.py
CHANGED
|
@@ -16,6 +16,7 @@ from __future__ import annotations
|
|
| 16 |
|
| 17 |
import os
|
| 18 |
import shutil
|
|
|
|
| 19 |
from pathlib import Path
|
| 20 |
|
| 21 |
# Default dataset when KINK_HF_REQUIRE_FULL_CATALOG=1 but Space UI cleared KINK_HF_DATASET_REPO (empty overrides Dockerfile).
|
|
@@ -97,6 +98,50 @@ def _download_store_from_http(url: str, dest: Path) -> None:
|
|
| 97 |
raise
|
| 98 |
|
| 99 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
def ensure_store_db(target: Path) -> Path:
|
| 101 |
"""Return ``target`` if it exists; otherwise download from Hub, or copy bundled seed when allowed."""
|
| 102 |
target = target.resolve()
|
|
|
|
| 16 |
|
| 17 |
import os
|
| 18 |
import shutil
|
| 19 |
+
import tarfile
|
| 20 |
from pathlib import Path
|
| 21 |
|
| 22 |
# Default dataset when KINK_HF_REQUIRE_FULL_CATALOG=1 but Space UI cleared KINK_HF_DATASET_REPO (empty overrides Dockerfile).
|
|
|
|
| 98 |
raise
|
| 99 |
|
| 100 |
|
| 101 |
+
def ensure_media_cache(target_root: Path) -> Path:
|
| 102 |
+
"""Hydrate ``data/cached_assets`` from a Hub dataset archive when configured."""
|
| 103 |
+
archive_name = os.environ.get("KINK_HF_MEDIA_ARCHIVE", "").strip()
|
| 104 |
+
if not archive_name:
|
| 105 |
+
return target_root
|
| 106 |
+
|
| 107 |
+
target_root = target_root.resolve()
|
| 108 |
+
marker = target_root / f".{archive_name.replace('/', '_')}.ready"
|
| 109 |
+
if marker.is_file():
|
| 110 |
+
return target_root
|
| 111 |
+
|
| 112 |
+
repo = (
|
| 113 |
+
os.environ.get("KINK_HF_MEDIA_DATASET_REPO", "").strip()
|
| 114 |
+
or os.environ.get("KINK_HF_DATASET_REPO", "").strip()
|
| 115 |
+
or DEFAULT_KINK_HF_DATASET_REPO
|
| 116 |
+
)
|
| 117 |
+
revision = os.environ.get("KINK_HF_MEDIA_DATASET_REVISION", "").strip() or None
|
| 118 |
+
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
|
| 119 |
+
if not repo:
|
| 120 |
+
raise RuntimeError("KINK_HF_MEDIA_ARCHIVE is set but no Hub dataset repo is configured")
|
| 121 |
+
|
| 122 |
+
try:
|
| 123 |
+
from huggingface_hub import hf_hub_download
|
| 124 |
+
except ImportError as exc:
|
| 125 |
+
raise ImportError(
|
| 126 |
+
"huggingface_hub is required when KINK_HF_MEDIA_ARCHIVE is set. "
|
| 127 |
+
"Install with: pip install huggingface_hub"
|
| 128 |
+
) from exc
|
| 129 |
+
|
| 130 |
+
target_root.mkdir(parents=True, exist_ok=True)
|
| 131 |
+
downloaded = hf_hub_download(
|
| 132 |
+
repo_id=repo,
|
| 133 |
+
repo_type="dataset",
|
| 134 |
+
filename=archive_name,
|
| 135 |
+
revision=revision,
|
| 136 |
+
token=token,
|
| 137 |
+
local_dir=str(target_root.parent),
|
| 138 |
+
)
|
| 139 |
+
with tarfile.open(downloaded, "r:*") as tar:
|
| 140 |
+
tar.extractall(target_root, filter="data")
|
| 141 |
+
marker.write_text("ok\n", encoding="utf-8")
|
| 142 |
+
return target_root
|
| 143 |
+
|
| 144 |
+
|
| 145 |
def ensure_store_db(target: Path) -> Path:
|
| 146 |
"""Return ``target`` if it exists; otherwise download from Hub, or copy bundled seed when allowed."""
|
| 147 |
target = target.resolve()
|
backend/media_remote.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Remote media URL resolution for catalog assets."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import os
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from functools import lru_cache
|
| 7 |
+
from urllib.parse import quote
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@dataclass(frozen=True)
|
| 11 |
+
class RemoteMediaConfig:
|
| 12 |
+
public_base_url: str
|
| 13 |
+
s3_bucket: str
|
| 14 |
+
s3_prefix: str
|
| 15 |
+
s3_endpoint_url: str
|
| 16 |
+
s3_region: str
|
| 17 |
+
s3_access_key_id: str
|
| 18 |
+
s3_secret_access_key: str
|
| 19 |
+
s3_expires_s: int
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def mode(self) -> str:
|
| 23 |
+
if self.public_base_url:
|
| 24 |
+
return "public_base"
|
| 25 |
+
if self.s3_bucket and self.s3_endpoint_url and self.s3_access_key_id and self.s3_secret_access_key:
|
| 26 |
+
return "s3_presign"
|
| 27 |
+
return "none"
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _env(name: str) -> str:
|
| 31 |
+
return (os.environ.get(name) or "").strip()
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _env_int(name: str, default: int, *, low: int, high: int) -> int:
|
| 35 |
+
raw = _env(name)
|
| 36 |
+
if not raw:
|
| 37 |
+
return default
|
| 38 |
+
value = int(raw)
|
| 39 |
+
return min(max(value, low), high)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _clean_prefix(prefix: str) -> str:
|
| 43 |
+
return prefix.strip().strip("/")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _object_key(prefix: str, relative_path: str) -> str:
|
| 47 |
+
rel = relative_path.strip().lstrip("/")
|
| 48 |
+
pref = _clean_prefix(prefix)
|
| 49 |
+
return f"{pref}/{rel}" if pref else rel
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _quote_path(path: str) -> str:
|
| 53 |
+
return "/".join(quote(part, safe="") for part in path.split("/"))
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def remote_media_config_from_env() -> RemoteMediaConfig:
|
| 57 |
+
public_base_url = _env("KINK_MEDIA_PUBLIC_BASE_URL").rstrip("/")
|
| 58 |
+
|
| 59 |
+
s3_prefix = _env("KINK_MEDIA_S3_PREFIX") or _env("KINK_B2_MEDIA_PREFIX")
|
| 60 |
+
explicit_s3 = bool(_env("KINK_MEDIA_S3_BUCKET"))
|
| 61 |
+
explicit_b2 = bool(_env("KINK_B2_MEDIA_BUCKET") or _env("KINK_B2_MEDIA_PREFIX"))
|
| 62 |
+
|
| 63 |
+
s3_bucket = _env("KINK_MEDIA_S3_BUCKET")
|
| 64 |
+
if not s3_bucket and explicit_b2:
|
| 65 |
+
s3_bucket = _env("KINK_B2_MEDIA_BUCKET") or _env("B2_BUCKET") or _env("KINK_B2_BUCKET")
|
| 66 |
+
|
| 67 |
+
s3_region = _env("KINK_MEDIA_S3_REGION")
|
| 68 |
+
if not s3_region and (explicit_b2 or s3_bucket):
|
| 69 |
+
s3_region = _env("KINK_B2_MEDIA_REGION") or _env("B2_REGION") or _env("KINK_B2_REGION")
|
| 70 |
+
|
| 71 |
+
s3_endpoint_url = _env("KINK_MEDIA_S3_ENDPOINT_URL")
|
| 72 |
+
if not s3_endpoint_url and explicit_b2 and s3_region:
|
| 73 |
+
s3_endpoint_url = f"https://s3.{s3_region}.backblazeb2.com"
|
| 74 |
+
|
| 75 |
+
s3_access_key_id = _env("KINK_MEDIA_S3_ACCESS_KEY_ID")
|
| 76 |
+
s3_secret_access_key = _env("KINK_MEDIA_S3_SECRET_ACCESS_KEY")
|
| 77 |
+
if explicit_b2:
|
| 78 |
+
s3_access_key_id = s3_access_key_id or _env("B2_KEY_ID") or _env("KINK_B2_KEY_ID")
|
| 79 |
+
s3_secret_access_key = (
|
| 80 |
+
s3_secret_access_key or _env("B2_APPLICATION_KEY") or _env("KINK_B2_APPLICATION_KEY")
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
if explicit_s3 and not s3_endpoint_url:
|
| 84 |
+
s3_endpoint_url = f"https://s3.{s3_region}.amazonaws.com" if s3_region else ""
|
| 85 |
+
|
| 86 |
+
return RemoteMediaConfig(
|
| 87 |
+
public_base_url=public_base_url,
|
| 88 |
+
s3_bucket=s3_bucket,
|
| 89 |
+
s3_prefix=s3_prefix,
|
| 90 |
+
s3_endpoint_url=s3_endpoint_url.rstrip("/"),
|
| 91 |
+
s3_region=s3_region,
|
| 92 |
+
s3_access_key_id=s3_access_key_id,
|
| 93 |
+
s3_secret_access_key=s3_secret_access_key,
|
| 94 |
+
s3_expires_s=_env_int("KINK_MEDIA_S3_EXPIRES_S", 3600, low=60, high=604800),
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@lru_cache(maxsize=8)
|
| 99 |
+
def _s3_client(endpoint_url: str, region: str, access_key_id: str, secret_access_key: str):
|
| 100 |
+
import boto3
|
| 101 |
+
|
| 102 |
+
return boto3.client(
|
| 103 |
+
"s3",
|
| 104 |
+
endpoint_url=endpoint_url,
|
| 105 |
+
aws_access_key_id=access_key_id,
|
| 106 |
+
aws_secret_access_key=secret_access_key,
|
| 107 |
+
region_name=region or None,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def remote_media_url(relative_path: str) -> str | None:
|
| 112 |
+
cfg = remote_media_config_from_env()
|
| 113 |
+
if cfg.mode == "public_base":
|
| 114 |
+
key = _quote_path(_object_key("", relative_path))
|
| 115 |
+
return f"{cfg.public_base_url}/{key}"
|
| 116 |
+
if cfg.mode == "s3_presign":
|
| 117 |
+
key = _object_key(cfg.s3_prefix, relative_path)
|
| 118 |
+
client = _s3_client(
|
| 119 |
+
cfg.s3_endpoint_url,
|
| 120 |
+
cfg.s3_region,
|
| 121 |
+
cfg.s3_access_key_id,
|
| 122 |
+
cfg.s3_secret_access_key,
|
| 123 |
+
)
|
| 124 |
+
return str(
|
| 125 |
+
client.generate_presigned_url(
|
| 126 |
+
"get_object",
|
| 127 |
+
Params={"Bucket": cfg.s3_bucket, "Key": key},
|
| 128 |
+
ExpiresIn=cfg.s3_expires_s,
|
| 129 |
+
)
|
| 130 |
+
)
|
| 131 |
+
return None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def remote_media_health_payload() -> dict[str, object]:
|
| 135 |
+
cfg = remote_media_config_from_env()
|
| 136 |
+
return {
|
| 137 |
+
"remote_mode": cfg.mode,
|
| 138 |
+
"remote_configured": cfg.mode != "none",
|
| 139 |
+
"remote_prefix": _clean_prefix(cfg.s3_prefix),
|
| 140 |
+
"remote_expires_s": cfg.s3_expires_s if cfg.mode == "s3_presign" else None,
|
| 141 |
+
}
|
frontend/components.js
CHANGED
|
@@ -3,7 +3,7 @@ import { html, UI_REACTIONS, DIRECTION_PILLS } from "./constants.js";
|
|
| 3 |
import { kinkCategoryLabel } from "./ui-labels.js";
|
| 4 |
|
| 5 |
/** One photo at a time; tap left/right edges to change. */
|
| 6 |
-
function KinkMediaScroll({ assets, imageRevealed, kinkName, onReveal, resetKey }) {
|
| 7 |
const [idx, setIdx] = useState(0);
|
| 8 |
const list = (Array.isArray(assets) ? assets : []).filter((a) => a && typeof a.asset_url === "string" && a.asset_url);
|
| 9 |
const n = list.length;
|
|
@@ -59,6 +59,7 @@ function KinkMediaScroll({ assets, imageRevealed, kinkName, onReveal, resetKey }
|
|
| 59 |
src=${cur.asset_url}
|
| 60 |
alt=""
|
| 61 |
draggable=${false}
|
|
|
|
| 62 |
/>
|
| 63 |
</div>
|
| 64 |
${n > 1
|
|
@@ -161,7 +162,21 @@ export function KinkCard({
|
|
| 161 |
onToggleInfo,
|
| 162 |
}) {
|
| 163 |
const assets = (kink.assets || []).filter((a) => a && typeof a.asset_url === "string" && a.asset_url);
|
| 164 |
-
const
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
const category = kinkCategoryLabel(kink);
|
| 166 |
const def = definitionText(kink);
|
| 167 |
const safety = html`
|
|
@@ -206,10 +221,11 @@ export function KinkCard({
|
|
| 206 |
<div className=${`tinder-card__media ${imageRevealed ? "revealed" : ""}`}>
|
| 207 |
<${KinkMediaScroll}
|
| 208 |
resetKey=${kink.id}
|
| 209 |
-
assets=${
|
| 210 |
imageRevealed=${imageRevealed}
|
| 211 |
kinkName=${kink.name}
|
| 212 |
onReveal=${onRevealImage}
|
|
|
|
| 213 |
/>
|
| 214 |
</div>
|
| 215 |
`
|
|
@@ -269,10 +285,15 @@ export function KinkCard({
|
|
| 269 |
? imageRevealed
|
| 270 |
? html`
|
| 271 |
<div className="card-image-scroll">
|
| 272 |
-
${
|
| 273 |
(asset, i) => html`
|
| 274 |
<div key=${asset.asset_url || `c-${i}`} className="card-image-slide">
|
| 275 |
-
<img
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
</div>
|
| 277 |
`,
|
| 278 |
)}
|
|
@@ -280,7 +301,13 @@ export function KinkCard({
|
|
| 280 |
`
|
| 281 |
: html`
|
| 282 |
<div className="card-image-area card-image-area--preview" onClick=${onRevealImage}>
|
| 283 |
-
<img
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 284 |
<span className="card-image-cta">Show</span>
|
| 285 |
</div>
|
| 286 |
`
|
|
|
|
| 3 |
import { kinkCategoryLabel } from "./ui-labels.js";
|
| 4 |
|
| 5 |
/** One photo at a time; tap left/right edges to change. */
|
| 6 |
+
function KinkMediaScroll({ assets, imageRevealed, kinkName, onReveal, onAssetError, resetKey }) {
|
| 7 |
const [idx, setIdx] = useState(0);
|
| 8 |
const list = (Array.isArray(assets) ? assets : []).filter((a) => a && typeof a.asset_url === "string" && a.asset_url);
|
| 9 |
const n = list.length;
|
|
|
|
| 59 |
src=${cur.asset_url}
|
| 60 |
alt=""
|
| 61 |
draggable=${false}
|
| 62 |
+
onError=${() => onAssetError?.(cur.asset_url)}
|
| 63 |
/>
|
| 64 |
</div>
|
| 65 |
${n > 1
|
|
|
|
| 162 |
onToggleInfo,
|
| 163 |
}) {
|
| 164 |
const assets = (kink.assets || []).filter((a) => a && typeof a.asset_url === "string" && a.asset_url);
|
| 165 |
+
const [failedAssetUrls, setFailedAssetUrls] = useState(() => new Set());
|
| 166 |
+
useEffect(() => {
|
| 167 |
+
setFailedAssetUrls(new Set());
|
| 168 |
+
}, [kink.id]);
|
| 169 |
+
const markAssetFailed = useCallback((assetUrl) => {
|
| 170 |
+
if (!assetUrl) return;
|
| 171 |
+
setFailedAssetUrls((prev) => {
|
| 172 |
+
if (prev.has(assetUrl)) return prev;
|
| 173 |
+
const next = new Set(prev);
|
| 174 |
+
next.add(assetUrl);
|
| 175 |
+
return next;
|
| 176 |
+
});
|
| 177 |
+
}, []);
|
| 178 |
+
const visibleAssets = assets.filter((a) => !failedAssetUrls.has(a.asset_url));
|
| 179 |
+
const hasImage = photosEnabled && visibleAssets.length > 0;
|
| 180 |
const category = kinkCategoryLabel(kink);
|
| 181 |
const def = definitionText(kink);
|
| 182 |
const safety = html`
|
|
|
|
| 221 |
<div className=${`tinder-card__media ${imageRevealed ? "revealed" : ""}`}>
|
| 222 |
<${KinkMediaScroll}
|
| 223 |
resetKey=${kink.id}
|
| 224 |
+
assets=${visibleAssets}
|
| 225 |
imageRevealed=${imageRevealed}
|
| 226 |
kinkName=${kink.name}
|
| 227 |
onReveal=${onRevealImage}
|
| 228 |
+
onAssetError=${markAssetFailed}
|
| 229 |
/>
|
| 230 |
</div>
|
| 231 |
`
|
|
|
|
| 285 |
? imageRevealed
|
| 286 |
? html`
|
| 287 |
<div className="card-image-scroll">
|
| 288 |
+
${visibleAssets.map(
|
| 289 |
(asset, i) => html`
|
| 290 |
<div key=${asset.asset_url || `c-${i}`} className="card-image-slide">
|
| 291 |
+
<img
|
| 292 |
+
loading=${i < 2 ? "eager" : "lazy"}
|
| 293 |
+
src=${asset.asset_url}
|
| 294 |
+
alt=""
|
| 295 |
+
onError=${() => markAssetFailed(asset.asset_url)}
|
| 296 |
+
/>
|
| 297 |
</div>
|
| 298 |
`,
|
| 299 |
)}
|
|
|
|
| 301 |
`
|
| 302 |
: html`
|
| 303 |
<div className="card-image-area card-image-area--preview" onClick=${onRevealImage}>
|
| 304 |
+
<img
|
| 305 |
+
className="card-image-blur"
|
| 306 |
+
src=${visibleAssets[0].asset_url}
|
| 307 |
+
alt=""
|
| 308 |
+
draggable=${false}
|
| 309 |
+
onError=${() => markAssetFailed(visibleAssets[0].asset_url)}
|
| 310 |
+
/>
|
| 311 |
<span className="card-image-cta">Show</span>
|
| 312 |
</div>
|
| 313 |
`
|
frontend/dist/assets/index-DoKOnBon.js
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
frontend/dist/index.html
CHANGED
|
@@ -5,7 +5,7 @@
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
<title>Play List</title>
|
| 7 |
<link rel="icon" href="/favicon.ico" type="image/jpeg" />
|
| 8 |
-
<script type="module" crossorigin src="/frontend/assets/index-
|
| 9 |
<link rel="stylesheet" crossorigin href="/frontend/assets/index-B1No5j6D.css">
|
| 10 |
</head>
|
| 11 |
<body>
|
|
|
|
| 5 |
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
| 6 |
<title>Play List</title>
|
| 7 |
<link rel="icon" href="/favicon.ico" type="image/jpeg" />
|
| 8 |
+
<script type="module" crossorigin src="/frontend/assets/index-DoKOnBon.js"></script>
|
| 9 |
<link rel="stylesheet" crossorigin href="/frontend/assets/index-B1No5j6D.css">
|
| 10 |
</head>
|
| 11 |
<body>
|
frontend/my-plays.js
CHANGED
|
@@ -262,11 +262,14 @@ function CollapsibleSection(props) {
|
|
| 262 |
function ImageGrid({ assets }) {
|
| 263 |
const [revealed, setRevealed] = useState(false);
|
| 264 |
const [idx, setIdx] = useState(0);
|
| 265 |
-
const
|
| 266 |
-
const
|
|
|
|
|
|
|
| 267 |
|
| 268 |
useEffect(() => {
|
| 269 |
setIdx(0);
|
|
|
|
| 270 |
}, [assets?.length, assets?.[0]?.asset_url]);
|
| 271 |
|
| 272 |
const go = useCallback((delta, e) => {
|
|
@@ -275,6 +278,22 @@ function ImageGrid({ assets }) {
|
|
| 275 |
setIdx((i) => (i + delta + n) % n);
|
| 276 |
}, [n]);
|
| 277 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
return html`
|
| 279 |
<div className="play-assets">
|
| 280 |
${!revealed
|
|
@@ -294,12 +313,19 @@ function ImageGrid({ assets }) {
|
|
| 294 |
`
|
| 295 |
: null}
|
| 296 |
<div className="play-assets-slide">
|
| 297 |
-
<img
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
</div>
|
| 299 |
${n > 1
|
| 300 |
? html`
|
| 301 |
<div className="play-assets-dots" aria-hidden="true">
|
| 302 |
-
${
|
| 303 |
</div>
|
| 304 |
<div className="play-assets-badge">${idx + 1} / ${n}</div>
|
| 305 |
`
|
|
|
|
| 262 |
function ImageGrid({ assets }) {
|
| 263 |
const [revealed, setRevealed] = useState(false);
|
| 264 |
const [idx, setIdx] = useState(0);
|
| 265 |
+
const [failedAssetUrls, setFailedAssetUrls] = useState(() => new Set());
|
| 266 |
+
const visibleAssets = assets.filter((asset) => !failedAssetUrls.has(asset.asset_url));
|
| 267 |
+
const n = visibleAssets.length;
|
| 268 |
+
const cur = visibleAssets[idx];
|
| 269 |
|
| 270 |
useEffect(() => {
|
| 271 |
setIdx(0);
|
| 272 |
+
setFailedAssetUrls(new Set());
|
| 273 |
}, [assets?.length, assets?.[0]?.asset_url]);
|
| 274 |
|
| 275 |
const go = useCallback((delta, e) => {
|
|
|
|
| 278 |
setIdx((i) => (i + delta + n) % n);
|
| 279 |
}, [n]);
|
| 280 |
|
| 281 |
+
const markAssetFailed = useCallback((assetUrl) => {
|
| 282 |
+
if (!assetUrl) return;
|
| 283 |
+
setFailedAssetUrls((prev) => {
|
| 284 |
+
if (prev.has(assetUrl)) return prev;
|
| 285 |
+
const next = new Set(prev);
|
| 286 |
+
next.add(assetUrl);
|
| 287 |
+
return next;
|
| 288 |
+
});
|
| 289 |
+
}, []);
|
| 290 |
+
|
| 291 |
+
useEffect(() => {
|
| 292 |
+
setIdx((i) => (n ? Math.min(Math.max(0, i), n - 1) : 0));
|
| 293 |
+
}, [n]);
|
| 294 |
+
|
| 295 |
+
if (!n) return null;
|
| 296 |
+
|
| 297 |
return html`
|
| 298 |
<div className="play-assets">
|
| 299 |
${!revealed
|
|
|
|
| 313 |
`
|
| 314 |
: null}
|
| 315 |
<div className="play-assets-slide">
|
| 316 |
+
<img
|
| 317 |
+
loading="eager"
|
| 318 |
+
decoding="async"
|
| 319 |
+
src=${cur.asset_url}
|
| 320 |
+
alt=""
|
| 321 |
+
draggable=${false}
|
| 322 |
+
onError=${() => markAssetFailed(cur.asset_url)}
|
| 323 |
+
/>
|
| 324 |
</div>
|
| 325 |
${n > 1
|
| 326 |
? html`
|
| 327 |
<div className="play-assets-dots" aria-hidden="true">
|
| 328 |
+
${visibleAssets.map((_, i) => html`<span key=${`pd-${i}`} className=${i === idx ? "on" : ""} />`)}
|
| 329 |
</div>
|
| 330 |
<div className="play-assets-badge">${idx + 1} / ${n}</div>
|
| 331 |
`
|
scripts/sync_hf_space_b2_media.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Upload cached catalog media to B2 and point the Hugging Face Space at it.
|
| 3 |
+
|
| 4 |
+
The app keeps stable same-origin ``/media/...`` URLs in the catalog. In production, when a
|
| 5 |
+
file is not present in the Space container, ``api.py`` can redirect that URL to a private
|
| 6 |
+
S3/B2 pre-signed URL. This script uploads ``data/cached_assets`` under one B2 prefix and
|
| 7 |
+
sets the Space variables/secrets required for that redirect path.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import concurrent.futures
|
| 13 |
+
import mimetypes
|
| 14 |
+
import os
|
| 15 |
+
import sys
|
| 16 |
+
import tomllib
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 20 |
+
if str(ROOT) not in sys.path:
|
| 21 |
+
sys.path.insert(0, str(ROOT))
|
| 22 |
+
|
| 23 |
+
from backend.b2_toml import load_b2_section, resolve_b2_config_path
|
| 24 |
+
from backend.repo_dotenv import load_repo_dotenv
|
| 25 |
+
|
| 26 |
+
DEFAULT_SECRETS = Path(
|
| 27 |
+
os.environ.get("KINK_PARSER_SECRETS_TOML", Path.home() / "PycharmProjects/parser/secrets.toml")
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _load_hf_token(path: Path) -> str:
|
| 32 |
+
token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip()
|
| 33 |
+
if token:
|
| 34 |
+
return token
|
| 35 |
+
with path.open("rb") as f:
|
| 36 |
+
data = tomllib.load(f)
|
| 37 |
+
token = str((data.get("huggingface") or {}).get("token") or "").strip()
|
| 38 |
+
if not token:
|
| 39 |
+
raise SystemExit(f"Missing [huggingface].token in {path}")
|
| 40 |
+
return token
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _iter_media_files(root: Path) -> list[Path]:
|
| 44 |
+
files = [p for p in root.rglob("*") if p.is_file()]
|
| 45 |
+
files.sort(key=lambda p: p.relative_to(root).as_posix())
|
| 46 |
+
return files
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _object_key(prefix: str, rel: str) -> str:
|
| 50 |
+
clean = prefix.strip().strip("/")
|
| 51 |
+
return f"{clean}/{rel}" if clean else rel
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _content_type(path: Path) -> str:
|
| 55 |
+
return mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def main() -> int:
|
| 59 |
+
load_repo_dotenv()
|
| 60 |
+
ap = argparse.ArgumentParser(description=__doc__)
|
| 61 |
+
ap.add_argument("--secrets-toml", type=Path, default=DEFAULT_SECRETS)
|
| 62 |
+
ap.add_argument("--b2-secrets-toml", type=Path, default=None)
|
| 63 |
+
ap.add_argument("--local-root", type=Path, default=ROOT / "data" / "cached_assets")
|
| 64 |
+
ap.add_argument("--bucket", default=None, help="Bucket override; defaults to [b2].bucket_name")
|
| 65 |
+
ap.add_argument("--prefix", default="kink/media", help="Object prefix for media files")
|
| 66 |
+
ap.add_argument("--workers", type=int, default=16)
|
| 67 |
+
ap.add_argument("--limit", type=int, default=0, help="Upload at most N files (test/dev)")
|
| 68 |
+
ap.add_argument("--skip-existing", action="store_true", help="HEAD each object and skip existing keys")
|
| 69 |
+
ap.add_argument("--skip-upload", action="store_true", help="Only push HF Space variables/secrets")
|
| 70 |
+
ap.add_argument("--no-space-update", action="store_true", help="Upload only; do not update HF Space")
|
| 71 |
+
ap.add_argument("--progress-every", type=int, default=1000)
|
| 72 |
+
args = ap.parse_args()
|
| 73 |
+
|
| 74 |
+
local_root = args.local_root.resolve()
|
| 75 |
+
if not local_root.is_dir():
|
| 76 |
+
print(f"sync_hf_space_b2_media: media root not found: {local_root}", file=sys.stderr)
|
| 77 |
+
return 1
|
| 78 |
+
if not args.secrets_toml.is_file():
|
| 79 |
+
print(f"sync_hf_space_b2_media: secrets not found: {args.secrets_toml}", file=sys.stderr)
|
| 80 |
+
return 1
|
| 81 |
+
b2_path = resolve_b2_config_path(cli_b2=args.b2_secrets_toml, cli_fallback=args.secrets_toml)
|
| 82 |
+
if not b2_path.is_file():
|
| 83 |
+
print(f"sync_hf_space_b2_media: B2 secrets TOML not found: {b2_path}", file=sys.stderr)
|
| 84 |
+
return 1
|
| 85 |
+
try:
|
| 86 |
+
b2 = load_b2_section(b2_path)
|
| 87 |
+
except SystemExit as exc:
|
| 88 |
+
print(f"sync_hf_space_b2_media: {exc}", file=sys.stderr)
|
| 89 |
+
return 1
|
| 90 |
+
|
| 91 |
+
try:
|
| 92 |
+
import boto3
|
| 93 |
+
from boto3.s3.transfer import TransferConfig
|
| 94 |
+
except ImportError:
|
| 95 |
+
print("sync_hf_space_b2_media: boto3 is required", file=sys.stderr)
|
| 96 |
+
return 1
|
| 97 |
+
|
| 98 |
+
region = b2["region"]
|
| 99 |
+
bucket = (args.bucket or os.environ.get("KINK_B2_MEDIA_BUCKET") or b2["bucket_name"]).strip()
|
| 100 |
+
endpoint = f"https://s3.{region}.backblazeb2.com"
|
| 101 |
+
client = boto3.client(
|
| 102 |
+
"s3",
|
| 103 |
+
endpoint_url=endpoint,
|
| 104 |
+
aws_access_key_id=b2["key_id"],
|
| 105 |
+
aws_secret_access_key=b2["application_key"],
|
| 106 |
+
region_name=region,
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
files = _iter_media_files(local_root)
|
| 110 |
+
if args.limit:
|
| 111 |
+
files = files[: args.limit]
|
| 112 |
+
total_bytes = sum(p.stat().st_size for p in files)
|
| 113 |
+
print(
|
| 114 |
+
f"sync_hf_space_b2_media: {len(files)} files, {total_bytes / 1e9:.2f} GB → s3://{bucket}/{args.prefix.strip().strip('/')}/",
|
| 115 |
+
file=sys.stderr,
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
if not args.skip_upload:
|
| 119 |
+
transfer_cfg = TransferConfig(use_threads=False)
|
| 120 |
+
|
| 121 |
+
def upload_one(path: Path) -> tuple[str, str]:
|
| 122 |
+
rel = path.relative_to(local_root).as_posix()
|
| 123 |
+
key = _object_key(args.prefix, rel)
|
| 124 |
+
if args.skip_existing:
|
| 125 |
+
try:
|
| 126 |
+
client.head_object(Bucket=bucket, Key=key)
|
| 127 |
+
return ("skipped", rel)
|
| 128 |
+
except Exception:
|
| 129 |
+
pass
|
| 130 |
+
client.upload_file(
|
| 131 |
+
str(path),
|
| 132 |
+
bucket,
|
| 133 |
+
key,
|
| 134 |
+
ExtraArgs={
|
| 135 |
+
"ContentType": _content_type(path),
|
| 136 |
+
"CacheControl": "public, max-age=31536000, immutable",
|
| 137 |
+
},
|
| 138 |
+
Config=transfer_cfg,
|
| 139 |
+
)
|
| 140 |
+
return ("uploaded", rel)
|
| 141 |
+
|
| 142 |
+
uploaded = 0
|
| 143 |
+
skipped = 0
|
| 144 |
+
with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
|
| 145 |
+
futures = [pool.submit(upload_one, path) for path in files]
|
| 146 |
+
for i, fut in enumerate(concurrent.futures.as_completed(futures), start=1):
|
| 147 |
+
status, rel = fut.result()
|
| 148 |
+
if status == "skipped":
|
| 149 |
+
skipped += 1
|
| 150 |
+
else:
|
| 151 |
+
uploaded += 1
|
| 152 |
+
if args.progress_every and (i % args.progress_every == 0 or i == len(futures)):
|
| 153 |
+
print(
|
| 154 |
+
f"sync_hf_space_b2_media: {i}/{len(futures)} done ({uploaded} uploaded, {skipped} skipped), last={rel}",
|
| 155 |
+
file=sys.stderr,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
if args.no_space_update:
|
| 159 |
+
return 0
|
| 160 |
+
|
| 161 |
+
repo_id = (os.environ.get("HF_SPACE_REPO") or "").strip()
|
| 162 |
+
if not repo_id:
|
| 163 |
+
print("sync_hf_space_b2_media: set HF_SPACE_REPO=owner/slug or pass --no-space-update", file=sys.stderr)
|
| 164 |
+
return 1
|
| 165 |
+
try:
|
| 166 |
+
hf_token = _load_hf_token(args.secrets_toml)
|
| 167 |
+
except SystemExit as exc:
|
| 168 |
+
print(f"sync_hf_space_b2_media: {exc}", file=sys.stderr)
|
| 169 |
+
return 1
|
| 170 |
+
|
| 171 |
+
from huggingface_hub import HfApi
|
| 172 |
+
|
| 173 |
+
api = HfApi(token=hf_token)
|
| 174 |
+
|
| 175 |
+
def sec(name: str, value: str, desc: str) -> None:
|
| 176 |
+
api.add_space_secret(repo_id, name, value, description=desc, token=hf_token)
|
| 177 |
+
|
| 178 |
+
sec("B2_KEY_ID", b2["key_id"], "Backblaze B2 application key id (S3)")
|
| 179 |
+
sec("B2_APPLICATION_KEY", b2["application_key"], "Backblaze B2 application key secret")
|
| 180 |
+
sec("B2_BUCKET", bucket, "Backblaze B2 bucket for catalog/media objects")
|
| 181 |
+
sec("B2_REGION", region, "Backblaze B2 region")
|
| 182 |
+
api.add_space_variable(
|
| 183 |
+
repo_id,
|
| 184 |
+
"KINK_B2_MEDIA_PREFIX",
|
| 185 |
+
args.prefix.strip().strip("/"),
|
| 186 |
+
description="B2/S3 prefix for catalog media files",
|
| 187 |
+
token=hf_token,
|
| 188 |
+
)
|
| 189 |
+
api.add_space_variable(
|
| 190 |
+
repo_id,
|
| 191 |
+
"KINK_MEDIA_S3_EXPIRES_S",
|
| 192 |
+
"21600",
|
| 193 |
+
description="Signed media URL lifetime in seconds",
|
| 194 |
+
token=hf_token,
|
| 195 |
+
)
|
| 196 |
+
api.add_space_variable(
|
| 197 |
+
repo_id,
|
| 198 |
+
"KINK_MEDIA_STRICT",
|
| 199 |
+
"1",
|
| 200 |
+
description="Do not serve demo placeholders for missing catalog media",
|
| 201 |
+
token=hf_token,
|
| 202 |
+
)
|
| 203 |
+
print("sync_hf_space_b2_media: Space media secrets/variables updated.", file=sys.stderr)
|
| 204 |
+
return 0
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
if __name__ == "__main__":
|
| 208 |
+
raise SystemExit(main())
|
scripts/verify_hf_stack.py
CHANGED
|
@@ -291,10 +291,16 @@ def _assert_catalog_https_image_fetchable(root: str) -> None:
|
|
| 291 |
print("verify_hf_stack: bundled fetlife_fetishes placeholder image ok", file=sys.stderr)
|
| 292 |
|
| 293 |
health = _get(f"{base}/health")
|
| 294 |
-
|
|
|
|
|
|
|
| 295 |
if n_assets < 1:
|
| 296 |
print("verify_hf_stack: skipping live catalog row probe (stats.assets is 0)", file=sys.stderr)
|
| 297 |
return
|
|
|
|
|
|
|
|
|
|
|
|
|
| 298 |
|
| 299 |
lst = _get(f"{base}/kinks?limit=500")
|
| 300 |
items = lst.get("items") or []
|
|
@@ -316,6 +322,10 @@ def _assert_catalog_https_image_fetchable(root: str) -> None:
|
|
| 316 |
assert_catalog_image_url_works(url, timeout_s=t_img)
|
| 317 |
except HTTPError as e:
|
| 318 |
if e.code == 404:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 319 |
print(
|
| 320 |
f"verify_hf_stack: catalog image GET 404 for {kid!r} (missing on-disk bytes for that path; "
|
| 321 |
"bundled seed images above still prove /media is wired).",
|
|
|
|
| 291 |
print("verify_hf_stack: bundled fetlife_fetishes placeholder image ok", file=sys.stderr)
|
| 292 |
|
| 293 |
health = _get(f"{base}/health")
|
| 294 |
+
stats = health.get("stats") or {}
|
| 295 |
+
media = health.get("media") or {}
|
| 296 |
+
n_assets = int(stats.get("assets") or 0)
|
| 297 |
if n_assets < 1:
|
| 298 |
print("verify_hf_stack: skipping live catalog row probe (stats.assets is 0)", file=sys.stderr)
|
| 299 |
return
|
| 300 |
+
require_catalog_image = (
|
| 301 |
+
os.environ.get("HF_VERIFY_REQUIRE_CATALOG_IMAGE", "").strip().lower() in ("1", "true", "yes", "on")
|
| 302 |
+
or (bool(media.get("strict_missing_assets")) and int(stats.get("kinks") or 0) >= 1000)
|
| 303 |
+
)
|
| 304 |
|
| 305 |
lst = _get(f"{base}/kinks?limit=500")
|
| 306 |
items = lst.get("items") or []
|
|
|
|
| 322 |
assert_catalog_image_url_works(url, timeout_s=t_img)
|
| 323 |
except HTTPError as e:
|
| 324 |
if e.code == 404:
|
| 325 |
+
if require_catalog_image:
|
| 326 |
+
raise RuntimeError(
|
| 327 |
+
f"catalog image GET 404 for {kid!r}; full-catalog media is required but bytes are missing"
|
| 328 |
+
) from e
|
| 329 |
print(
|
| 330 |
f"verify_hf_stack: catalog image GET 404 for {kid!r} (missing on-disk bytes for that path; "
|
| 331 |
"bundled seed images above still prove /media is wired).",
|
tests/test_api_media_fallback.py
CHANGED
|
@@ -55,6 +55,30 @@ def test_missing_media_strict_returns_404(tmp_path, monkeypatch) -> None:
|
|
| 55 |
assert r.status_code == 404
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
def test_favicon_serves_placeholder(media_fallback_app) -> None:
|
| 59 |
client = TestClient(media_fallback_app.app)
|
| 60 |
r = client.get("/favicon.ico")
|
|
@@ -62,8 +86,8 @@ def test_favicon_serves_placeholder(media_fallback_app) -> None:
|
|
| 62 |
assert r.headers.get("content-type", "").startswith("image/")
|
| 63 |
|
| 64 |
|
| 65 |
-
def
|
| 66 |
-
"""Published HF image marker
|
| 67 |
assert _SEED.is_file(), f"Missing seed DB: {_SEED}"
|
| 68 |
dest = tmp_path / "media_hf_strict.db"
|
| 69 |
shutil.copyfile(_SEED, dest)
|
|
@@ -79,12 +103,33 @@ def test_hf_space_image_ignores_media_strict_for_missing_tiles(tmp_path, monkeyp
|
|
| 79 |
api_mod._get_backend()
|
| 80 |
client = TestClient(api_mod.app)
|
| 81 |
r = client.get("/media/fetlife_fetishes/999999/0000000000001.jpg")
|
| 82 |
-
assert r.status_code ==
|
| 83 |
-
assert r.headers.get("x-kink-media-fallback") == "1"
|
| 84 |
h = client.get("/health").json()
|
| 85 |
assert h.get("media", {}).get("hf_space_image") is True
|
| 86 |
assert h.get("media", {}).get("strict_env") is True
|
| 87 |
-
assert h.get("media", {}).get("strict_missing_assets") is
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
|
| 90 |
def test_invalid_kink_media_fallback_override_still_serves_bundled_demo(tmp_path, monkeypatch) -> None:
|
|
|
|
| 55 |
assert r.status_code == 404
|
| 56 |
|
| 57 |
|
| 58 |
+
def test_missing_media_redirects_to_public_media_base(tmp_path, monkeypatch) -> None:
|
| 59 |
+
assert _SEED.is_file(), f"Missing seed DB: {_SEED}"
|
| 60 |
+
dest = tmp_path / "remote_media_store.db"
|
| 61 |
+
shutil.copyfile(_SEED, dest)
|
| 62 |
+
monkeypatch.setenv("KINK_STORE_PATH", str(dest))
|
| 63 |
+
monkeypatch.setenv("KINK_SKIP_HEAVY_WARM", "1")
|
| 64 |
+
monkeypatch.setenv("KINK_HF_REQUIRE_FULL_CATALOG", "1")
|
| 65 |
+
monkeypatch.setenv("KINK_HF_STALE_MAX_BYTES", "0")
|
| 66 |
+
monkeypatch.setenv("KINK_MEDIA_PUBLIC_BASE_URL", "https://cdn.example.test/kink-media")
|
| 67 |
+
monkeypatch.delenv("KINK_MEDIA_STRICT", raising=False)
|
| 68 |
+
sys.modules.pop("api", None)
|
| 69 |
+
import api as api_mod
|
| 70 |
+
|
| 71 |
+
api_mod._backend_impl = None
|
| 72 |
+
api_mod._get_backend()
|
| 73 |
+
client = TestClient(api_mod.app)
|
| 74 |
+
r = client.get("/media/fetlife_fetishes/999999/0000000000001.jpg", follow_redirects=False)
|
| 75 |
+
assert r.status_code == 307
|
| 76 |
+
assert r.headers["location"] == "https://cdn.example.test/kink-media/fetlife_fetishes/999999/0000000000001.jpg"
|
| 77 |
+
h = client.get("/health").json()
|
| 78 |
+
assert h.get("media", {}).get("remote_mode") == "public_base"
|
| 79 |
+
assert h.get("media", {}).get("remote_configured") is True
|
| 80 |
+
|
| 81 |
+
|
| 82 |
def test_favicon_serves_placeholder(media_fallback_app) -> None:
|
| 83 |
client = TestClient(media_fallback_app.app)
|
| 84 |
r = client.get("/favicon.ico")
|
|
|
|
| 86 |
assert r.headers.get("content-type", "").startswith("image/")
|
| 87 |
|
| 88 |
|
| 89 |
+
def test_hf_space_image_honors_media_strict_for_missing_tiles(tmp_path, monkeypatch) -> None:
|
| 90 |
+
"""Published HF image marker must not override an explicit Space ``KINK_MEDIA_STRICT=1``."""
|
| 91 |
assert _SEED.is_file(), f"Missing seed DB: {_SEED}"
|
| 92 |
dest = tmp_path / "media_hf_strict.db"
|
| 93 |
shutil.copyfile(_SEED, dest)
|
|
|
|
| 103 |
api_mod._get_backend()
|
| 104 |
client = TestClient(api_mod.app)
|
| 105 |
r = client.get("/media/fetlife_fetishes/999999/0000000000001.jpg")
|
| 106 |
+
assert r.status_code == 404
|
|
|
|
| 107 |
h = client.get("/health").json()
|
| 108 |
assert h.get("media", {}).get("hf_space_image") is True
|
| 109 |
assert h.get("media", {}).get("strict_env") is True
|
| 110 |
+
assert h.get("media", {}).get("strict_missing_assets") is True
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def test_full_catalog_mode_defaults_to_strict_missing_media(tmp_path, monkeypatch) -> None:
|
| 114 |
+
assert _SEED.is_file(), f"Missing seed DB: {_SEED}"
|
| 115 |
+
dest = tmp_path / "media_full_catalog.db"
|
| 116 |
+
shutil.copyfile(_SEED, dest)
|
| 117 |
+
monkeypatch.setenv("KINK_STORE_PATH", str(dest))
|
| 118 |
+
monkeypatch.setenv("KINK_SKIP_HEAVY_WARM", "1")
|
| 119 |
+
monkeypatch.setenv("KINK_HF_REQUIRE_FULL_CATALOG", "1")
|
| 120 |
+
monkeypatch.setenv("KINK_HF_STALE_MAX_BYTES", "0")
|
| 121 |
+
monkeypatch.delenv("KINK_MEDIA_STRICT", raising=False)
|
| 122 |
+
sys.modules.pop("api", None)
|
| 123 |
+
import api as api_mod
|
| 124 |
+
|
| 125 |
+
api_mod._backend_impl = None
|
| 126 |
+
api_mod._get_backend()
|
| 127 |
+
client = TestClient(api_mod.app)
|
| 128 |
+
r = client.get("/media/fetlife_fetishes/999999/0000000000001.jpg")
|
| 129 |
+
assert r.status_code == 404
|
| 130 |
+
h = client.get("/health").json()
|
| 131 |
+
assert h.get("media", {}).get("strict_env") is False
|
| 132 |
+
assert h.get("media", {}).get("strict_missing_assets") is True
|
| 133 |
|
| 134 |
|
| 135 |
def test_invalid_kink_media_fallback_override_still_serves_bundled_demo(tmp_path, monkeypatch) -> None:
|
tests/test_hf_bootstrap.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
"""Persistence and optional Hub bootstrap behavior."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
from unittest.mock import MagicMock, patch
|
| 6 |
|
|
@@ -30,6 +31,40 @@ def test_ensure_store_db_uses_existing_file(tmp_path: Path):
|
|
| 30 |
assert ensure_store_db(p) == p.resolve()
|
| 31 |
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
def test_ensure_store_db_require_full_replaces_tiny_existing_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
| 34 |
"""Stale bundled seed on disk must not block Hub download when full catalog is required."""
|
| 35 |
from backend import hf_bootstrap
|
|
|
|
| 1 |
"""Persistence and optional Hub bootstrap behavior."""
|
| 2 |
from __future__ import annotations
|
| 3 |
|
| 4 |
+
import tarfile
|
| 5 |
from pathlib import Path
|
| 6 |
from unittest.mock import MagicMock, patch
|
| 7 |
|
|
|
|
| 31 |
assert ensure_store_db(p) == p.resolve()
|
| 32 |
|
| 33 |
|
| 34 |
+
def test_ensure_media_cache_downloads_and_extracts_archive(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
| 35 |
+
from backend import hf_bootstrap
|
| 36 |
+
|
| 37 |
+
remote_dir = tmp_path / "remote"
|
| 38 |
+
source_dir = tmp_path / "source"
|
| 39 |
+
source_dir.mkdir()
|
| 40 |
+
(source_dir / "fetlife_fetishes").mkdir()
|
| 41 |
+
(source_dir / "fetlife_fetishes" / "1.jpg").write_bytes(b"jpeg")
|
| 42 |
+
archive = remote_dir / "cached_assets.tar"
|
| 43 |
+
remote_dir.mkdir()
|
| 44 |
+
with tarfile.open(archive, "w") as tar:
|
| 45 |
+
tar.add(source_dir / "fetlife_fetishes" / "1.jpg", arcname="fetlife_fetishes/1.jpg")
|
| 46 |
+
|
| 47 |
+
seen: dict[str, str] = {}
|
| 48 |
+
|
| 49 |
+
def fake_download(*, repo_id, filename, revision, token, local_dir, repo_type, **kwargs):
|
| 50 |
+
seen["repo_id"] = repo_id
|
| 51 |
+
seen["filename"] = filename
|
| 52 |
+
dest = Path(local_dir) / filename
|
| 53 |
+
dest.write_bytes(archive.read_bytes())
|
| 54 |
+
return str(dest)
|
| 55 |
+
|
| 56 |
+
target = tmp_path / "cached_assets"
|
| 57 |
+
monkeypatch.setenv("KINK_HF_MEDIA_ARCHIVE", "cached_assets.tar")
|
| 58 |
+
monkeypatch.setenv("KINK_HF_MEDIA_DATASET_REPO", "dummy/media")
|
| 59 |
+
with patch("huggingface_hub.hf_hub_download", side_effect=fake_download):
|
| 60 |
+
out = hf_bootstrap.ensure_media_cache(target)
|
| 61 |
+
|
| 62 |
+
assert out == target.resolve()
|
| 63 |
+
assert seen == {"repo_id": "dummy/media", "filename": "cached_assets.tar"}
|
| 64 |
+
assert (target / "fetlife_fetishes" / "1.jpg").read_bytes() == b"jpeg"
|
| 65 |
+
assert (target / ".cached_assets.tar.ready").is_file()
|
| 66 |
+
|
| 67 |
+
|
| 68 |
def test_ensure_store_db_require_full_replaces_tiny_existing_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
| 69 |
"""Stale bundled seed on disk must not block Hub download when full catalog is required."""
|
| 70 |
from backend import hf_bootstrap
|