Ox1 Cursor commited on
Commit
edbf640
·
1 Parent(s): 75e0882

feat(storage): add dual storage abstraction (local/S3)

Browse files

Add src/storage.py with two backends:
- local: writes to data/garments/ (default for development)
- s3: uses HF Spaces S3 bucket (for production persistence)

Backend is selected via STORAGE_BACKEND env var. Includes .env.example
with all configurable variables documented.

Co-authored-by: Cursor <cursoragent@cursor.com>

Files changed (3) hide show
  1. .env.example +12 -0
  2. requirements.txt +1 -0
  3. src/storage.py +141 -0
.env.example ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Storage backend: "local" (default) or "s3"
2
+ STORAGE_BACKEND=local
3
+
4
+ # S3 configuration (only needed when STORAGE_BACKEND=s3)
5
+ S3_BUCKET_NAME=
6
+ S3_ENDPOINT_URL=
7
+ S3_PREFIX=garments/
8
+ AWS_ACCESS_KEY_ID=
9
+ AWS_SECRET_ACCESS_KEY=
10
+
11
+ # GPU configuration
12
+ CUDA_VISIBLE_DEVICES=0
requirements.txt CHANGED
@@ -2,3 +2,4 @@ gradio==6.17.3
2
  llama-cpp-python>=0.3.28
3
  huggingface-hub>=1.18.0
4
  Pillow>=12.0.0
 
 
2
  llama-cpp-python>=0.3.28
3
  huggingface-hub>=1.18.0
4
  Pillow>=12.0.0
5
+ boto3>=1.35.0
src/storage.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Storage abstraction layer for garment images and data.
2
+
3
+ Supports two backends controlled by STORAGE_BACKEND env var:
4
+ - "local" (default): filesystem at data/garments/
5
+ - "s3": HuggingFace Spaces S3 bucket for persistent storage
6
+ """
7
+
8
+ import io
9
+ import logging
10
+ import os
11
+ from pathlib import Path
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ STORAGE_BACKEND = os.environ.get("STORAGE_BACKEND", "local")
16
+ LOCAL_GARMENTS_DIR = Path(__file__).parent.parent / "data" / "garments"
17
+
18
+ S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", "")
19
+ S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "")
20
+ S3_PREFIX = os.environ.get("S3_PREFIX", "garments/")
21
+
22
+ _s3_client = None
23
+
24
+
25
+ def _get_s3_client():
26
+ global _s3_client
27
+ if _s3_client is None:
28
+ import boto3
29
+
30
+ kwargs = {}
31
+ if S3_ENDPOINT_URL:
32
+ kwargs["endpoint_url"] = S3_ENDPOINT_URL
33
+
34
+ _s3_client = boto3.client("s3", **kwargs)
35
+ return _s3_client
36
+
37
+
38
+ def _ensure_local_dir():
39
+ LOCAL_GARMENTS_DIR.mkdir(parents=True, exist_ok=True)
40
+
41
+
42
+ def save_image(garment_id: str, image_bytes: bytes, extension: str = "jpg") -> str:
43
+ """Save a garment image and return its storage reference.
44
+
45
+ The reference is backend-agnostic (just the filename). Use
46
+ get_image_url() to resolve it to a serveable path/URL.
47
+ """
48
+ filename = f"{garment_id}.{extension}"
49
+
50
+ if STORAGE_BACKEND == "s3":
51
+ _save_to_s3(filename, image_bytes)
52
+ else:
53
+ _save_to_local(filename, image_bytes)
54
+
55
+ logger.info("Saved image: %s (backend: %s)", filename, STORAGE_BACKEND)
56
+ return filename
57
+
58
+
59
+ def _save_to_local(filename: str, image_bytes: bytes):
60
+ _ensure_local_dir()
61
+ path = LOCAL_GARMENTS_DIR / filename
62
+ path.write_bytes(image_bytes)
63
+
64
+
65
+ def _save_to_s3(filename: str, image_bytes: bytes):
66
+ client = _get_s3_client()
67
+ key = f"{S3_PREFIX}{filename}"
68
+ client.put_object(
69
+ Bucket=S3_BUCKET_NAME,
70
+ Key=key,
71
+ Body=image_bytes,
72
+ ContentType="image/jpeg",
73
+ )
74
+
75
+
76
+ def get_image_path(garment_id: str, extension: str = "jpg") -> str | None:
77
+ """Return the local filesystem path for a garment image.
78
+
79
+ For S3 backend, downloads to a local cache first.
80
+ Returns None if the image doesn't exist.
81
+ """
82
+ filename = f"{garment_id}.{extension}"
83
+
84
+ if STORAGE_BACKEND == "s3":
85
+ return _download_from_s3(filename)
86
+
87
+ path = LOCAL_GARMENTS_DIR / filename
88
+ if path.exists():
89
+ return str(path)
90
+ return None
91
+
92
+
93
+ def _download_from_s3(filename: str) -> str | None:
94
+ _ensure_local_dir()
95
+ local_path = LOCAL_GARMENTS_DIR / filename
96
+
97
+ if local_path.exists():
98
+ return str(local_path)
99
+
100
+ try:
101
+ client = _get_s3_client()
102
+ key = f"{S3_PREFIX}{filename}"
103
+ response = client.get_object(Bucket=S3_BUCKET_NAME, Key=key)
104
+ local_path.write_bytes(response["Body"].read())
105
+ return str(local_path)
106
+ except Exception as e:
107
+ logger.warning("Failed to download %s from S3: %s", filename, e)
108
+ return None
109
+
110
+
111
+ def delete_image(garment_id: str, extension: str = "jpg"):
112
+ """Remove a garment image from storage."""
113
+ filename = f"{garment_id}.{extension}"
114
+
115
+ if STORAGE_BACKEND == "s3":
116
+ try:
117
+ client = _get_s3_client()
118
+ key = f"{S3_PREFIX}{filename}"
119
+ client.delete_object(Bucket=S3_BUCKET_NAME, Key=key)
120
+ except Exception as e:
121
+ logger.warning("Failed to delete %s from S3: %s", filename, e)
122
+
123
+ local_path = LOCAL_GARMENTS_DIR / filename
124
+ if local_path.exists():
125
+ local_path.unlink()
126
+
127
+
128
+ def image_exists(garment_id: str, extension: str = "jpg") -> bool:
129
+ """Check if a garment image exists in storage."""
130
+ filename = f"{garment_id}.{extension}"
131
+
132
+ if STORAGE_BACKEND == "s3":
133
+ try:
134
+ client = _get_s3_client()
135
+ key = f"{S3_PREFIX}{filename}"
136
+ client.head_object(Bucket=S3_BUCKET_NAME, Key=key)
137
+ return True
138
+ except Exception:
139
+ return False
140
+
141
+ return (LOCAL_GARMENTS_DIR / filename).exists()