Spaces:
Runtime error
Runtime error
feat(storage): add dual storage abstraction (local/S3)
Browse filesAdd src/storage.py with two backends:
- local: writes to data/garments/ (default for development)
- s3: uses HF Spaces S3 bucket (for production persistence)
Backend is selected via STORAGE_BACKEND env var. Includes .env.example
with all configurable variables documented.
Co-authored-by: Cursor <cursoragent@cursor.com>
- .env.example +12 -0
- requirements.txt +1 -0
- src/storage.py +141 -0
.env.example
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Storage backend: "local" (default) or "s3"
|
| 2 |
+
STORAGE_BACKEND=local
|
| 3 |
+
|
| 4 |
+
# S3 configuration (only needed when STORAGE_BACKEND=s3)
|
| 5 |
+
S3_BUCKET_NAME=
|
| 6 |
+
S3_ENDPOINT_URL=
|
| 7 |
+
S3_PREFIX=garments/
|
| 8 |
+
AWS_ACCESS_KEY_ID=
|
| 9 |
+
AWS_SECRET_ACCESS_KEY=
|
| 10 |
+
|
| 11 |
+
# GPU configuration
|
| 12 |
+
CUDA_VISIBLE_DEVICES=0
|
requirements.txt
CHANGED
|
@@ -2,3 +2,4 @@ gradio==6.17.3
|
|
| 2 |
llama-cpp-python>=0.3.28
|
| 3 |
huggingface-hub>=1.18.0
|
| 4 |
Pillow>=12.0.0
|
|
|
|
|
|
| 2 |
llama-cpp-python>=0.3.28
|
| 3 |
huggingface-hub>=1.18.0
|
| 4 |
Pillow>=12.0.0
|
| 5 |
+
boto3>=1.35.0
|
src/storage.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Storage abstraction layer for garment images and data.
|
| 2 |
+
|
| 3 |
+
Supports two backends controlled by STORAGE_BACKEND env var:
|
| 4 |
+
- "local" (default): filesystem at data/garments/
|
| 5 |
+
- "s3": HuggingFace Spaces S3 bucket for persistent storage
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import io
|
| 9 |
+
import logging
|
| 10 |
+
import os
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
STORAGE_BACKEND = os.environ.get("STORAGE_BACKEND", "local")
|
| 16 |
+
LOCAL_GARMENTS_DIR = Path(__file__).parent.parent / "data" / "garments"
|
| 17 |
+
|
| 18 |
+
S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", "")
|
| 19 |
+
S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "")
|
| 20 |
+
S3_PREFIX = os.environ.get("S3_PREFIX", "garments/")
|
| 21 |
+
|
| 22 |
+
_s3_client = None
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _get_s3_client():
|
| 26 |
+
global _s3_client
|
| 27 |
+
if _s3_client is None:
|
| 28 |
+
import boto3
|
| 29 |
+
|
| 30 |
+
kwargs = {}
|
| 31 |
+
if S3_ENDPOINT_URL:
|
| 32 |
+
kwargs["endpoint_url"] = S3_ENDPOINT_URL
|
| 33 |
+
|
| 34 |
+
_s3_client = boto3.client("s3", **kwargs)
|
| 35 |
+
return _s3_client
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _ensure_local_dir():
|
| 39 |
+
LOCAL_GARMENTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def save_image(garment_id: str, image_bytes: bytes, extension: str = "jpg") -> str:
|
| 43 |
+
"""Save a garment image and return its storage reference.
|
| 44 |
+
|
| 45 |
+
The reference is backend-agnostic (just the filename). Use
|
| 46 |
+
get_image_url() to resolve it to a serveable path/URL.
|
| 47 |
+
"""
|
| 48 |
+
filename = f"{garment_id}.{extension}"
|
| 49 |
+
|
| 50 |
+
if STORAGE_BACKEND == "s3":
|
| 51 |
+
_save_to_s3(filename, image_bytes)
|
| 52 |
+
else:
|
| 53 |
+
_save_to_local(filename, image_bytes)
|
| 54 |
+
|
| 55 |
+
logger.info("Saved image: %s (backend: %s)", filename, STORAGE_BACKEND)
|
| 56 |
+
return filename
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _save_to_local(filename: str, image_bytes: bytes):
|
| 60 |
+
_ensure_local_dir()
|
| 61 |
+
path = LOCAL_GARMENTS_DIR / filename
|
| 62 |
+
path.write_bytes(image_bytes)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _save_to_s3(filename: str, image_bytes: bytes):
|
| 66 |
+
client = _get_s3_client()
|
| 67 |
+
key = f"{S3_PREFIX}{filename}"
|
| 68 |
+
client.put_object(
|
| 69 |
+
Bucket=S3_BUCKET_NAME,
|
| 70 |
+
Key=key,
|
| 71 |
+
Body=image_bytes,
|
| 72 |
+
ContentType="image/jpeg",
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def get_image_path(garment_id: str, extension: str = "jpg") -> str | None:
|
| 77 |
+
"""Return the local filesystem path for a garment image.
|
| 78 |
+
|
| 79 |
+
For S3 backend, downloads to a local cache first.
|
| 80 |
+
Returns None if the image doesn't exist.
|
| 81 |
+
"""
|
| 82 |
+
filename = f"{garment_id}.{extension}"
|
| 83 |
+
|
| 84 |
+
if STORAGE_BACKEND == "s3":
|
| 85 |
+
return _download_from_s3(filename)
|
| 86 |
+
|
| 87 |
+
path = LOCAL_GARMENTS_DIR / filename
|
| 88 |
+
if path.exists():
|
| 89 |
+
return str(path)
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _download_from_s3(filename: str) -> str | None:
|
| 94 |
+
_ensure_local_dir()
|
| 95 |
+
local_path = LOCAL_GARMENTS_DIR / filename
|
| 96 |
+
|
| 97 |
+
if local_path.exists():
|
| 98 |
+
return str(local_path)
|
| 99 |
+
|
| 100 |
+
try:
|
| 101 |
+
client = _get_s3_client()
|
| 102 |
+
key = f"{S3_PREFIX}{filename}"
|
| 103 |
+
response = client.get_object(Bucket=S3_BUCKET_NAME, Key=key)
|
| 104 |
+
local_path.write_bytes(response["Body"].read())
|
| 105 |
+
return str(local_path)
|
| 106 |
+
except Exception as e:
|
| 107 |
+
logger.warning("Failed to download %s from S3: %s", filename, e)
|
| 108 |
+
return None
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def delete_image(garment_id: str, extension: str = "jpg"):
|
| 112 |
+
"""Remove a garment image from storage."""
|
| 113 |
+
filename = f"{garment_id}.{extension}"
|
| 114 |
+
|
| 115 |
+
if STORAGE_BACKEND == "s3":
|
| 116 |
+
try:
|
| 117 |
+
client = _get_s3_client()
|
| 118 |
+
key = f"{S3_PREFIX}{filename}"
|
| 119 |
+
client.delete_object(Bucket=S3_BUCKET_NAME, Key=key)
|
| 120 |
+
except Exception as e:
|
| 121 |
+
logger.warning("Failed to delete %s from S3: %s", filename, e)
|
| 122 |
+
|
| 123 |
+
local_path = LOCAL_GARMENTS_DIR / filename
|
| 124 |
+
if local_path.exists():
|
| 125 |
+
local_path.unlink()
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def image_exists(garment_id: str, extension: str = "jpg") -> bool:
|
| 129 |
+
"""Check if a garment image exists in storage."""
|
| 130 |
+
filename = f"{garment_id}.{extension}"
|
| 131 |
+
|
| 132 |
+
if STORAGE_BACKEND == "s3":
|
| 133 |
+
try:
|
| 134 |
+
client = _get_s3_client()
|
| 135 |
+
key = f"{S3_PREFIX}{filename}"
|
| 136 |
+
client.head_object(Bucket=S3_BUCKET_NAME, Key=key)
|
| 137 |
+
return True
|
| 138 |
+
except Exception:
|
| 139 |
+
return False
|
| 140 |
+
|
| 141 |
+
return (LOCAL_GARMENTS_DIR / filename).exists()
|