Upload folder using huggingface_hub

script/storage_recompress_oneshot.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+"""One-shot Supabase storage recompress (v7 "hot cache, low-res").
+
+Context: the `ring-measurements` bucket crossed the 1 GB free-tier quota.
+The measurement pipeline never sees past ~1024px (SAM downscales internally),
+so storing full-res user uploads buys nothing for re-measurement. This script
+shrinks the existing objects in place:
+
+  - photos/   -> 2048px long-edge JPEG q80   (validated measurement-safe)
+  - results/  -> 1024px long-edge JPEG q80   (overlay, never re-measured)
+
+Safety model (free tier has NO rollback):
+  1. `backup` downloads every photos/ object to a local dir FIRST and leaves
+     it pristine. That is the ultimate fallback for photos.
+  2. Recompress encodes in memory, RE-DECODES to verify the bytes are a valid
+     image, and only then overwrites. Never writes garbage over a good object.
+  3. Photos are recompressed only if a verified local backup exists.
+  4. skip-if-not-smaller: never write a candidate that isn't meaningfully
+     smaller than what's already there (avoids re-encode bloat).
+
+Subcommands:
+    inventory                 read-only: count + bytes for photos/ + results/
+    backup                    download all photos/ -> BACKUP_DIR (idempotent)
+    test-mechanism            upload+overwrite+verify+delete a throwaway object
+    recompress --prefix photos  --max-side 2048 [--apply]
+    recompress --prefix results --max-side 1024 [--apply]
+
+Without --apply, recompress is a DRY RUN (projects savings, writes nothing).
+
+Requires SUPABASE_URL / SUPABASE_SERVICE_KEY in the environment.
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import cv2
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from web_demo.supabase_client import _get_client, BUCKET  # noqa: E402
+
+BACKUP_DIR = ROOT / "input" / "supabase_archive"
+PAGE_LIMIT = 100
+JPEG_QUALITY = 80
+MIN_GAIN = 0.95  # only write if candidate <= 95% of original size
+
+
+# --------------------------------------------------------------------------- #
+# storage helpers
+# --------------------------------------------------------------------------- #
+def _storage():
+    client = _get_client()
+    if client is None:
+        print("ERROR: Supabase client not initialized "
+              "(check SUPABASE_URL / SUPABASE_SERVICE_KEY).")
+        sys.exit(1)
+    return client.storage.from_(BUCKET)
+
+
+def _obj_size(obj: dict) -> Optional[int]:
+    md = obj.get("metadata") or {}
+    return md.get("size") if md.get("size") is not None else obj.get("size")
+
+
+def list_all(storage, folder: str) -> List[dict]:
+    out: List[dict] = []
+    offset = 0
+    while True:
+        resp = storage.list(folder, {"limit": PAGE_LIMIT, "offset": offset})
+        if not resp:
+            break
+        # storage may return a placeholder row for the folder itself; drop dirs
+        out.extend([o for o in resp if o.get("name") and _obj_size(o) is not None])
+        if len(resp) < PAGE_LIMIT:
+            break
+        offset += PAGE_LIMIT
+    return out
+
+
+def _human(n: float) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if abs(n) < 1024:
+            return f"{n:.1f} {unit}"
+        n /= 1024
+    return f"{n:.1f} TB"
+
+
+def _overwrite(storage, path: str, data: bytes, content_type: str) -> None:
+    """Overwrite an existing object in place. Tries update(), falls back to
+    upload() with x-upsert."""
+    opts = {"content-type": content_type, "cache-control": "3600"}
+    try:
+        storage.update(path, data, file_options=opts)
+    except Exception:
+        storage.upload(path, data,
+                       file_options={**opts, "upsert": "true"})
+
+
+# --------------------------------------------------------------------------- #
+# encode
+# --------------------------------------------------------------------------- #
+def recompress_bytes(raw: bytes, max_side: int) -> Optional[bytes]:
+    """Decode -> (optionally) downscale to max_side -> JPEG q80 -> re-decode
+    verify. Returns new bytes, or None if decode/verify failed."""
+    arr = np.frombuffer(raw, dtype=np.uint8)
+    img = cv2.imdecode(arr, cv2.IMREAD_COLOR)  # drops alpha; BGR
+    if img is None:
+        return None
+    h, w = img.shape[:2]
+    long_side = max(h, w)
+    if long_side > max_side:
+        scale = max_side / long_side
+        img = cv2.resize(img, (int(round(w * scale)), int(round(h * scale))),
+                         interpolation=cv2.INTER_AREA)
+    ok, enc = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, JPEG_QUALITY])
+    if not ok:
+        return None
+    out = enc.tobytes()
+    # verify: the new bytes must decode back to a valid image
+    if cv2.imdecode(np.frombuffer(out, np.uint8), cv2.IMREAD_COLOR) is None:
+        return None
+    return out
+
+
+# --------------------------------------------------------------------------- #
+# subcommands
+# --------------------------------------------------------------------------- #
+def cmd_inventory(_args) -> int:
+    storage = _storage()
+    grand = 0
+    for folder in ("photos", "results", "feedback"):
+        objs = list_all(storage, folder)
+        total = sum(_obj_size(o) or 0 for o in objs)
+        grand += total
+        print(f"{folder}/: {len(objs)} objects, {_human(total)}")
+    print(f"TOTAL (3 prefixes): {_human(grand)}")
+    return 0
+
+
+def cmd_backup(_args) -> int:
+    storage = _storage()
+    dest = BACKUP_DIR / "photos"
+    dest.mkdir(parents=True, exist_ok=True)
+    objs = list_all(storage, "photos")
+    print(f"photos/: {len(objs)} objects to back up -> {dest}")
+    done = skipped = failed = 0
+    for i, o in enumerate(objs, 1):
+        name = o["name"]
+        size = _obj_size(o) or 0
+        local = dest / name
+        if local.exists() and local.stat().st_size == size:
+            skipped += 1
+            continue
+        try:
+            data = storage.download(f"photos/{name}")
+            local.write_bytes(data)
+            if local.stat().st_size != size:
+                print(f"  [{i}] SIZE MISMATCH {name}: "
+                      f"got {len(data)} expected {size}")
+                failed += 1
+                continue
+            done += 1
+            if done % 25 == 0:
+                print(f"  ...{done} downloaded")
+        except Exception as e:
+            print(f"  [{i}] FAILED {name}: {e}")
+            failed += 1
+    print(f"backup done: {done} new, {skipped} already present, {failed} failed")
+    return 1 if failed else 0
+
+
+def cmd_test_mechanism(_args) -> int:
+    storage = _storage()
+    path = "photos/__recompress_selftest__.jpg"
+    a = cv2.imencode(".jpg", np.full((64, 64, 3), 30, np.uint8))[1].tobytes()
+    b = cv2.imencode(".jpg", np.full((64, 64, 3), 200, np.uint8))[1].tobytes()
+    try:
+        try:
+            storage.remove([path])
+        except Exception:
+            pass
+        storage.upload(path, a, file_options={"content-type": "image/jpeg"})
+        _overwrite(storage, path, b, "image/jpeg")
+        back = storage.download(path)
+        ok = (back == b)
+        print(f"overwrite-in-place {'OK' if ok else 'FAILED'} "
+              f"(round-tripped {len(back)} bytes, expected {len(b)})")
+        return 0 if ok else 1
+    finally:
+        try:
+            storage.remove([path])
+        except Exception:
+            pass
+
+
+def cmd_recompress(args) -> int:
+    storage = _storage()
+    prefix = args.prefix
+    max_side = args.max_side
+    apply = args.apply
+    objs = list_all(storage, prefix)
+    print(f"{prefix}/: {len(objs)} objects, target {max_side}px/q{JPEG_QUALITY}, "
+          f"{'APPLY' if apply else 'DRY RUN'}")
+
+    backup_photos = BACKUP_DIR / "photos"
+    before = after = 0
+    written = skipped_small = skipped_nobackup = failed = 0
+
+    for i, o in enumerate(objs, 1):
+        name = o["name"]
+        if name.startswith("__recompress_selftest__"):
+            continue
+        size = _obj_size(o) or 0
+        before += size
+        path = f"{prefix}/{name}"
+
+        # photos: require a verified local backup before destructive write
+        if prefix == "photos":
+            b = backup_photos / name
+            if not (b.exists() and b.stat().st_size == size):
+                skipped_nobackup += 1
+                after += size
+                continue
+
+        try:
+            raw = storage.download(path)
+        except Exception as e:
+            print(f"  [{i}] download FAILED {name}: {e}")
+            failed += 1
+            after += size
+            continue
+
+        new = recompress_bytes(raw, max_side)
+        if new is None:
+            print(f"  [{i}] decode/verify FAILED {name} — left untouched")
+            failed += 1
+            after += size
+            continue
+
+        if len(new) >= size * MIN_GAIN:
+            skipped_small += 1
+            after += size
+            continue
+
+        after += len(new)
+        if apply:
+            try:
+                _overwrite(storage, path, new, "image/jpeg")
+                written += 1
+                if written % 25 == 0:
+                    print(f"  ...{written} rewritten")
+            except Exception as e:
+                print(f"  [{i}] upload FAILED {name}: {e}")
+                failed += 1
+                after += size - len(new)  # revert projection
+        else:
+            written += 1  # would-write count in dry run
+
+    verb = "rewrote" if apply else "would rewrite"
+    print(f"\n{prefix}/ summary:")
+    print(f"  {verb}:            {written}")
+    print(f"  skipped (not smaller): {skipped_small}")
+    if prefix == "photos":
+        print(f"  skipped (no backup):   {skipped_nobackup}")
+    print(f"  failed:                {failed}")
+    print(f"  size: {_human(before)} -> {_human(after)} "
+          f"({(1 - after / before) * 100:.1f}% smaller)" if before else "  size: 0")
+    return 1 if failed else 0
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+    sub.add_parser("inventory")
+    sub.add_parser("backup")
+    sub.add_parser("test-mechanism")
+    r = sub.add_parser("recompress")
+    r.add_argument("--prefix", required=True, choices=("photos", "results"))
+    r.add_argument("--max-side", type=int, required=True)
+    r.add_argument("--apply", action="store_true",
+                   help="actually overwrite (default: dry run)")
+    args = p.parse_args()
+    return {
+        "inventory": cmd_inventory,
+        "backup": cmd_backup,
+        "test-mechanism": cmd_test_mechanism,
+        "recompress": cmd_recompress,
+    }[args.cmd](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

web_demo/app.py

@@ -35,6 +35,7 @@ from src.ring_size import recommend_ring_size, RING_MODELS, VALID_RING_MODELS, D
 from src.ai_recommendation import ai_explain_recommendation
 from web_demo.supabase_client import (
     upload_file,
+    upload_bytes,
     save_measurement,
     save_feedback,
     persistence_enabled,
@@ -50,6 +51,7 @@ from web_demo.supabase_client import (
     update_ground_truth,
     delete_measurement,
     delete_feedback,
+    storage_usage,
 )
 from src.confidence_constants import (
     CONFIDENCE_LEVEL_HIGH_THRESHOLD,
@@ -79,6 +81,50 @@ logger = logging.getLogger(__name__)
 # request queues one task, and we don't need ordering.
 _persist_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="supa-persist")
 
+# v7 storage diet: shrink what we store, not what we measure. Measurement
+# runs on the full-res in-memory image FIRST; only the stored copies are
+# downscaled. The pipeline never sees past ~1024px anyway (SAM downscales
+# internally), so a 2048px stored photo reproduces the full-res measurement
+# (validated: median 0.07 mm delta, zero new failures at 2048/q80). The
+# result overlay is display-only and never re-measured, so it goes smaller.
+PHOTO_MAX_SIDE = 2048
+RESULT_MAX_SIDE = 1024
+STORE_JPEG_QUALITY = 80
+
+
+def _shrink_jpeg(image_path: Path, max_side: int) -> Optional[bytes]:
+    """Read an image, downscale to max_side long-edge (never upscale), and
+    return JPEG q80 bytes. Returns None on any decode/encode failure so the
+    caller can fall back to uploading the original file."""
+    try:
+        img = cv2.imread(str(image_path))
+        if img is None:
+            return None
+        h, w = img.shape[:2]
+        long_side = max(h, w)
+        if long_side > max_side:
+            scale = max_side / long_side
+            img = cv2.resize(img, (int(round(w * scale)), int(round(h * scale))),
+                             interpolation=cv2.INTER_AREA)
+        ok, enc = cv2.imencode(".jpg", img,
+                               [cv2.IMWRITE_JPEG_QUALITY, STORE_JPEG_QUALITY])
+        if not ok:
+            return None
+        return enc.tobytes()
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("shrink failed for %s: %s", image_path, exc)
+        return None
+
+
+def _upload_shrunk(local_path: Path, storage_path: str, max_side: int) -> Optional[str]:
+    """Upload a downscaled JPEG copy of local_path. Falls back to uploading
+    the original file if shrink fails, so a recompress hiccup never drops the
+    image entirely."""
+    data = _shrink_jpeg(local_path, max_side)
+    if data is not None:
+        return upload_bytes(data, storage_path, "image/jpeg")
+    return upload_file(str(local_path), storage_path)
+
 
 def _persist_measurement_async(
     *,
@@ -98,9 +144,14 @@ def _persist_measurement_async(
             photo_url = None
             result_url = None
             if upload_path and upload_path.exists():
-                photo_url = upload_file(str(upload_path), f"photos/{upload_name}")
+                photo_url = _upload_shrunk(
+                    upload_path, f"photos/{upload_name}", PHOTO_MAX_SIDE)
             if result_png_path.exists():
-                result_url = upload_file(str(result_png_path), f"results/{result_png_name}")
+                # Keep the historical `_result.png` object name (DB result_url
+                # points at it) but store JPEG bytes — browsers honor the
+                # image/jpeg content-type over the .png extension.
+                result_url = _upload_shrunk(
+                    result_png_path, f"results/{result_png_name}", RESULT_MAX_SIDE)
             record_with_urls = dict(record)
             record_with_urls["photo_url"] = photo_url
             record_with_urls["result_url"] = result_url
@@ -972,6 +1023,17 @@ def api_admin_stats():
     return jsonify(_compute_stats(rows, days=days))
 
 
+@app.route("/api/admin/storage")
+def api_admin_storage():
+    if not _check_admin_token():
+        return jsonify({"error": "Unauthorized"}), 401
+    usage = storage_usage()
+    if usage is None:
+        return jsonify({"available": False})
+    usage["available"] = True
+    return jsonify(usage)
+
+
 _CSV_INJECTION_LEAD = ("=", "+", "-", "@", "\t", "\r")
 
 

web_demo/supabase_client.py

@@ -64,6 +64,29 @@ def persistence_enabled() -> bool:
 BUCKET = "ring-measurements"
 
 
+def upload_bytes(data: bytes, storage_path: str, content_type: str) -> Optional[str]:
+    """Upload raw bytes to Supabase Storage. Returns public URL or None.
+
+    Used by the web demo to upload an in-memory downscaled JPEG (v7 storage
+    diet) without first writing it to a temp file. content_type is explicit
+    because the stored object name may keep a legacy extension (e.g. a
+    `_result.png` object that now carries JPEG bytes)."""
+    client = _get_client()
+    if client is None:
+        return None
+    try:
+        client.storage.from_(BUCKET).upload(
+            storage_path,
+            data,
+            file_options={"content-type": content_type},
+        )
+        logger.info("Storage upload %s: %s bytes", storage_path, len(data))
+        return client.storage.from_(BUCKET).get_public_url(storage_path)
+    except Exception as e:
+        logger.error("Failed to upload %s (%s bytes): %s", storage_path, len(data), e)
+        return None
+
+
 def upload_file(local_path: str, storage_path: str) -> Optional[str]:
     """Upload a file to Supabase Storage. Returns public URL or None."""
     client = _get_client()
@@ -204,6 +227,64 @@ def count_feedback() -> Optional[int]:
         return None
 
 
+def _list_prefix_objects(storage, prefix: str) -> List[Dict[str, Any]]:
+    """Paginate storage.list(prefix) and return every real object (skips the
+    folder placeholder row that has no size metadata)."""
+    out: List[Dict[str, Any]] = []
+    offset = 0
+    page = 100
+    while True:
+        resp = storage.list(prefix, {"limit": page, "offset": offset})
+        if not resp:
+            break
+        for o in resp:
+            md = o.get("metadata") or {}
+            if md.get("size") is not None:
+                out.append(o)
+        if len(resp) < page:
+            break
+        offset += page
+    return out
+
+
+# Free-tier storage quota; used only to compute a headroom % for the admin
+# panel. Supabase enforces 1 GB; we surface headroom against a slightly
+# conservative 1.0 GB so the number nudges before the hard wall.
+STORAGE_QUOTA_BYTES = 1024 * 1024 * 1024
+
+
+def storage_usage() -> Optional[Dict[str, Any]]:
+    """Summarize bucket usage for the admin dashboard: per-prefix object count
+    + bytes, grand total, and headroom vs the free-tier quota. Returns None
+    when persistence is disabled so the UI can show 'unknown' rather than 0.
+
+    Walks the storage list API (the bucket has no size column); a few hundred
+    objects across three prefixes is one quick paginated sweep per prefix.
+    """
+    client = _get_client()
+    if client is None:
+        return None
+    try:
+        storage = client.storage.from_(BUCKET)
+        prefixes = {}
+        total = 0
+        for prefix in ("photos", "results", "feedback"):
+            objs = _list_prefix_objects(storage, prefix)
+            size = sum((o.get("metadata") or {}).get("size", 0) for o in objs)
+            prefixes[prefix] = {"objects": len(objs), "bytes": size}
+            total += size
+        return {
+            "prefixes": prefixes,
+            "total_bytes": total,
+            "quota_bytes": STORAGE_QUOTA_BYTES,
+            "used_fraction": (total / STORAGE_QUOTA_BYTES
+                              if STORAGE_QUOTA_BYTES else None),
+        }
+    except Exception as e:
+        logger.error("Failed to compute storage usage: %s", e)
+        return None
+
+
 def list_measurements(limit: int = 200, offset: int = 0) -> List[Dict[str, Any]]:
     """Fetch measurements for admin page, newest first."""
     client = _get_client()

web_demo/templates/admin.html

@@ -334,6 +334,7 @@
       adminContent.style.display = "block";
       document.getElementById("exportCsvLink").href = `/api/admin/export-csv?token=${encodeURIComponent(adminToken)}`;
       loadStats();
+      loadStorage();
       loadData();
       loadFeedback();
       return true;
@@ -610,6 +611,31 @@
 
     const fmtPct = (v) => v == null ? "-" : (v * 100).toFixed(1) + "%";
     const fmtInt = (v) => v == null ? "-" : v.toLocaleString();
+    const fmtBytes = (n) => {
+      if (n == null) return "—";
+      const u = ["B", "KB", "MB", "GB", "TB"];
+      let i = 0;
+      while (n >= 1024 && i < u.length - 1) { n /= 1024; i++; }
+      return n.toFixed(n < 10 && i > 0 ? 1 : 0) + " " + u[i];
+    };
+
+    // Storage usage is fetched separately from /api/admin/stats; cached here
+    // and folded into the stat-card grid so it sits with the headline numbers.
+    let storageStats = null;
+    let lastStats = null;
+
+    const storageCard = () => {
+      if (storageStats === null) return { label: "Storage used", value: "…", sub: "" };
+      if (!storageStats.available) return { label: "Storage used", value: "n/a", sub: "persistence off" };
+      const s = storageStats;
+      const pct = s.used_fraction != null ? (s.used_fraction * 100).toFixed(0) + "%" : "";
+      const p = s.prefixes || {};
+      const part = (k) => (p[k] ? `${k} ${fmtBytes(p[k].bytes)}` : "");
+      const sub = `${pct} of ${fmtBytes(s.quota_bytes)} · ` +
+        [part("photos"), part("results"), part("feedback")].filter(Boolean).join(" · ");
+      const subCls = (s.used_fraction != null && s.used_fraction > 0.8) ? "delta-down" : "";
+      return { label: "Storage used", value: fmtBytes(s.total_bytes), sub, subCls };
+    };
     const fmtDay = (iso) => {
       if (!iso) return "—";
       // YYYY-MM-DD → MM-DD for compact axis labels
@@ -626,6 +652,7 @@
     };
 
     const renderStatCards = (s) => {
+      lastStats = s;
       const t = s.totals;
       const last7Delta = deltaArrow(t.last_7_days, t.prev_7_days);
       const cards = [
@@ -639,6 +666,7 @@
           value: t.rating_count ? `${t.avg_rating.toFixed(2)} ★` : "—",
           sub: `${fmtInt(t.rating_count)} rated · ${fmtInt(t.comment_count)} comments`,
         },
+        storageCard(),
       ];
       statGrid.innerHTML = cards.map((c) => `
         <div class="stat-card">
@@ -786,7 +814,18 @@
       }
     };
 
-    document.getElementById("dashRefreshBtn").addEventListener("click", loadStats);
+    const loadStorage = async () => {
+      try {
+        const resp = await fetch(`/api/admin/storage?token=${encodeURIComponent(adminToken)}`);
+        if (resp.status === 401) return;
+        storageStats = await resp.json();
+        if (lastStats) renderStatCards(lastStats);  // re-render so the card fills in
+      } catch (e) {
+        console.error("Failed to load storage", e);
+      }
+    };
+
+    document.getElementById("dashRefreshBtn").addEventListener("click", () => { loadStats(); loadStorage(); });
     windowSelect.addEventListener("change", loadStats);
   </script>
 </body>