Spaces:
Running
Running
doc: Refactor and Refining code
Browse files- src/common/utils.py +7 -6
- src/core/config.py +30 -15
- src/core/logging.py +20 -18
src/common/utils.py
CHANGED
|
@@ -2,20 +2,20 @@ import re
|
|
| 2 |
import math
|
| 3 |
from fastapi import Request
|
| 4 |
|
| 5 |
-
|
| 6 |
def get_ip(request: Request) -> str:
|
| 7 |
forwarded = request.headers.get("X-Forwarded-For")
|
| 8 |
if forwarded:
|
| 9 |
return forwarded.split(",")[0].strip()
|
| 10 |
return request.client.host if request.client else "unknown"
|
| 11 |
|
| 12 |
-
|
| 13 |
def is_default_key(key: str, default: str) -> bool:
|
| 14 |
if not key or not default:
|
| 15 |
return False
|
| 16 |
return key.strip() == default.strip()
|
| 17 |
|
| 18 |
-
|
| 19 |
def get_cloudinary_creds(url: str) -> dict:
|
| 20 |
if not url or not url.startswith("cloudinary://"):
|
| 21 |
return {}
|
|
@@ -31,13 +31,13 @@ def get_cloudinary_creds(url: str) -> dict:
|
|
| 31 |
except ValueError:
|
| 32 |
return {}
|
| 33 |
|
| 34 |
-
|
| 35 |
def sanitize_filename(filename: str) -> str:
|
| 36 |
if not filename:
|
| 37 |
return "unnamed_file"
|
| 38 |
return re.sub(r'[^a-zA-Z0-9_\-\.]', '_', filename)
|
| 39 |
|
| 40 |
-
|
| 41 |
def standardize_category_name(name: str) -> str:
|
| 42 |
if not name:
|
| 43 |
return "uncategorized"
|
|
@@ -52,7 +52,7 @@ def to_list(vector) -> list[float]:
|
|
| 52 |
except TypeError:
|
| 53 |
return []
|
| 54 |
|
| 55 |
-
|
| 56 |
def url_to_public_id(url: str) -> str:
|
| 57 |
if not url:
|
| 58 |
return ""
|
|
@@ -60,6 +60,7 @@ def url_to_public_id(url: str) -> str:
|
|
| 60 |
parts = url.split("/upload/")
|
| 61 |
if len(parts) > 1:
|
| 62 |
path = parts[1].split("/", 1)[-1]
|
|
|
|
| 63 |
return path.rsplit(".", 1)[0]
|
| 64 |
return ""
|
| 65 |
except Exception:
|
|
|
|
| 2 |
import math
|
| 3 |
from fastapi import Request
|
| 4 |
|
| 5 |
+
# Get the Real IP of the User not the infra IP (if behind proxy) for logging and abuse monitoring.
|
| 6 |
def get_ip(request: Request) -> str:
|
| 7 |
forwarded = request.headers.get("X-Forwarded-For")
|
| 8 |
if forwarded:
|
| 9 |
return forwarded.split(",")[0].strip()
|
| 10 |
return request.client.host if request.client else "unknown"
|
| 11 |
|
| 12 |
+
# Returns Boolean indicating if the provided key is the same as the default value (ignoring whitespace).
|
| 13 |
def is_default_key(key: str, default: str) -> bool:
|
| 14 |
if not key or not default:
|
| 15 |
return False
|
| 16 |
return key.strip() == default.strip()
|
| 17 |
|
| 18 |
+
# Takes The Cloduinary URL and extracts the credentials from it and returns a dict with cloud_name, api_key and api_secret.
|
| 19 |
def get_cloudinary_creds(url: str) -> dict:
|
| 20 |
if not url or not url.startswith("cloudinary://"):
|
| 21 |
return {}
|
|
|
|
| 31 |
except ValueError:
|
| 32 |
return {}
|
| 33 |
|
| 34 |
+
# Replace everything except letters, numbers, _, -, . with underscores
|
| 35 |
def sanitize_filename(filename: str) -> str:
|
| 36 |
if not filename:
|
| 37 |
return "unnamed_file"
|
| 38 |
return re.sub(r'[^a-zA-Z0-9_\-\.]', '_', filename)
|
| 39 |
|
| 40 |
+
# Lower Casing the File Name
|
| 41 |
def standardize_category_name(name: str) -> str:
|
| 42 |
if not name:
|
| 43 |
return "uncategorized"
|
|
|
|
| 52 |
except TypeError:
|
| 53 |
return []
|
| 54 |
|
| 55 |
+
# Extracting the ID from The URL
|
| 56 |
def url_to_public_id(url: str) -> str:
|
| 57 |
if not url:
|
| 58 |
return ""
|
|
|
|
| 60 |
parts = url.split("/upload/")
|
| 61 |
if len(parts) > 1:
|
| 62 |
path = parts[1].split("/", 1)[-1]
|
| 63 |
+
# Splits From Right and place that thing first in list
|
| 64 |
return path.rsplit(".", 1)[0]
|
| 65 |
return ""
|
| 66 |
except Exception:
|
src/core/config.py
CHANGED
|
@@ -3,32 +3,48 @@ from dotenv import load_dotenv
|
|
| 3 |
|
| 4 |
load_dotenv()
|
| 5 |
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
DEFAULT_PINECONE_KEY = os.getenv("DEFAULT_PINECONE_KEY", "")
|
| 10 |
DEFAULT_CLOUDINARY_URL = os.getenv("DEFAULT_CLOUDINARY_URL", "")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
-
#
|
| 13 |
IDX_FACES = os.getenv("IDX_FACES", "enterprise-faces")
|
| 14 |
IDX_OBJECTS = os.getenv("IDX_OBJECTS", "enterprise-objects")
|
| 15 |
|
| 16 |
-
#
|
| 17 |
IDX_FACES_ARCFACE = os.getenv("IDX_FACES_ARCFACE", "faces-arcface")
|
| 18 |
IDX_FACES_ADAFACE = os.getenv("IDX_FACES_ADAFACE", "faces-adaface")
|
| 19 |
|
| 20 |
# ──────────────────────────────────────────────────────────────
|
| 21 |
-
# Concurrency / limits
|
| 22 |
# ──────────────────────────────────────────────────────────────
|
| 23 |
MAX_CONCURRENT_INFERENCES = int(os.getenv("MAX_CONCURRENT_INFERENCES", "2"))
|
| 24 |
MAX_FILES_PER_UPLOAD = int(os.getenv("MAX_FILES_PER_UPLOAD", "50"))
|
| 25 |
INFERENCE_CACHE_SIZE = int(os.getenv("INFERENCE_CACHE_SIZE", "128"))
|
| 26 |
|
| 27 |
-
|
| 28 |
-
# Logging
|
| 29 |
-
# ──────────────────────────────────────────────────────────────
|
| 30 |
-
SUPABASE_URL = os.getenv("SUPABASE_URL", "")
|
| 31 |
-
SUPABASE_SERVICE_KEY = os.getenv("SUPABASE_SERVICE_KEY", "")
|
| 32 |
|
| 33 |
# ──────────────────────────────────────────────────────────────
|
| 34 |
# Image / detection
|
|
@@ -58,7 +74,7 @@ FACE_BLUR_THRESHOLD = float(os.getenv("FACE_BLUR_THRESHOLD", "50.0"))
|
|
| 58 |
# ──────────────────────────────────────────────────────────────
|
| 59 |
FACE_DIM = 512
|
| 60 |
ADAFACE_DIM = 512
|
| 61 |
-
|
| 62 |
|
| 63 |
FACE_CROP_THUMB_SIZE = int(os.getenv("FACE_CROP_THUMB_SIZE", "112"))
|
| 64 |
FACE_CROP_QUALITY = int(os.getenv("FACE_CROP_QUALITY", "85"))
|
|
@@ -66,7 +82,7 @@ FACE_CROP_PADDING = float(os.getenv("FACE_CROP_PADDING", "0.2"))
|
|
| 66 |
ADAFACE_CROP_PADDING = float(os.getenv("ADAFACE_CROP_PADDING", "0.1"))
|
| 67 |
|
| 68 |
ENABLE_ADAFACE = int(os.getenv("ENABLE_ADAFACE", "1"))
|
| 69 |
-
|
| 70 |
|
| 71 |
# ──────────────────────────────────────────────────────────────
|
| 72 |
# Phase 1: Speed flags (unchanged, leaving on)
|
|
@@ -120,8 +136,7 @@ FACE_RESULTS_PER_QUERY_CAP = int(os.getenv("FACE_RESULTS_PER_QUERY_CAP", "200"))
|
|
| 120 |
|
| 121 |
# Redis-backed inference cache + job queue (requires Upstash)
|
| 122 |
# Set UPSTASH_REDIS_URL + UPSTASH_REDIS_TOKEN in HF Space secrets.
|
| 123 |
-
|
| 124 |
-
UPSTASH_REDIS_TOKEN = os.getenv("UPSTASH_REDIS_TOKEN", "")
|
| 125 |
|
| 126 |
# Master toggle: enable the persistent Redis cache (replaces in-memory dict).
|
| 127 |
# Falls back to in-memory if UPSTASH_REDIS_URL is not set, so this is safe to
|
|
|
|
| 3 |
|
| 4 |
load_dotenv()
|
| 5 |
|
| 6 |
+
"""This Files have Imported and created Global Variables which are used
|
| 7 |
+
Throughout the Project , There are in Total of 56 Variables
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
# ===============================================================
|
| 12 |
+
# Credentals / Secrets (set in HF Space secrets) / KEYS / URLS
|
| 13 |
+
# ===============================================================
|
| 14 |
+
|
| 15 |
+
#Pinecone + Cloudinary credentials (set in HF Space secrets)
|
| 16 |
DEFAULT_PINECONE_KEY = os.getenv("DEFAULT_PINECONE_KEY", "")
|
| 17 |
DEFAULT_CLOUDINARY_URL = os.getenv("DEFAULT_CLOUDINARY_URL", "")
|
| 18 |
+
#supbase Credentials
|
| 19 |
+
SUPABASE_URL = os.getenv("SUPABASE_URL", "")
|
| 20 |
+
SUPABASE_SERVICE_KEY = os.getenv("SUPABASE_SERVICE_KEY", "")
|
| 21 |
+
# HF Credentials
|
| 22 |
+
HF_TOKEN = os.getenv("HF_TOKEN", "")
|
| 23 |
+
# Upstash_Redis credentials
|
| 24 |
+
UPSTASH_REDIS_URL = os.getenv("UPSTASH_REDIS_URL", "")
|
| 25 |
+
UPSTASH_REDIS_TOKEN = os.getenv("UPSTASH_REDIS_TOKEN", "")
|
| 26 |
+
|
| 27 |
+
# ===============================================================
|
| 28 |
+
# Variables Set in HF Spaces Can be configured
|
| 29 |
+
# ===============================================================
|
| 30 |
+
|
| 31 |
|
| 32 |
+
# Index Names To be used to name variable for storing embeddings genereted from DInov2 and SIGLIP models
|
| 33 |
IDX_FACES = os.getenv("IDX_FACES", "enterprise-faces")
|
| 34 |
IDX_OBJECTS = os.getenv("IDX_OBJECTS", "enterprise-objects")
|
| 35 |
|
| 36 |
+
# Configure Index Names for storing Embedding Generated by ARCFACE and ADAFACE
|
| 37 |
IDX_FACES_ARCFACE = os.getenv("IDX_FACES_ARCFACE", "faces-arcface")
|
| 38 |
IDX_FACES_ADAFACE = os.getenv("IDX_FACES_ADAFACE", "faces-adaface")
|
| 39 |
|
| 40 |
# ──────────────────────────────────────────────────────────────
|
| 41 |
+
# Concurrency / limits / For Speeding Up Inference
|
| 42 |
# ──────────────────────────────────────────────────────────────
|
| 43 |
MAX_CONCURRENT_INFERENCES = int(os.getenv("MAX_CONCURRENT_INFERENCES", "2"))
|
| 44 |
MAX_FILES_PER_UPLOAD = int(os.getenv("MAX_FILES_PER_UPLOAD", "50"))
|
| 45 |
INFERENCE_CACHE_SIZE = int(os.getenv("INFERENCE_CACHE_SIZE", "128"))
|
| 46 |
|
| 47 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
# ──────────────────────────────────────────────────────────────
|
| 50 |
# Image / detection
|
|
|
|
| 74 |
# ──────────────────────────────────────────────────────────────
|
| 75 |
FACE_DIM = 512
|
| 76 |
ADAFACE_DIM = 512
|
| 77 |
+
|
| 78 |
|
| 79 |
FACE_CROP_THUMB_SIZE = int(os.getenv("FACE_CROP_THUMB_SIZE", "112"))
|
| 80 |
FACE_CROP_QUALITY = int(os.getenv("FACE_CROP_QUALITY", "85"))
|
|
|
|
| 82 |
ADAFACE_CROP_PADDING = float(os.getenv("ADAFACE_CROP_PADDING", "0.1"))
|
| 83 |
|
| 84 |
ENABLE_ADAFACE = int(os.getenv("ENABLE_ADAFACE", "1"))
|
| 85 |
+
|
| 86 |
|
| 87 |
# ──────────────────────────────────────────────────────────────
|
| 88 |
# Phase 1: Speed flags (unchanged, leaving on)
|
|
|
|
| 136 |
|
| 137 |
# Redis-backed inference cache + job queue (requires Upstash)
|
| 138 |
# Set UPSTASH_REDIS_URL + UPSTASH_REDIS_TOKEN in HF Space secrets.
|
| 139 |
+
|
|
|
|
| 140 |
|
| 141 |
# Master toggle: enable the persistent Redis cache (replaces in-memory dict).
|
| 142 |
# Falls back to in-memory if UPSTASH_REDIS_URL is not set, so this is safe to
|
src/core/logging.py
CHANGED
|
@@ -1,10 +1,26 @@
|
|
| 1 |
import asyncio
|
| 2 |
import json
|
| 3 |
-
|
| 4 |
from src.core.config import SUPABASE_URL, SUPABASE_SERVICE_KEY
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
_http_session = None
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
async def init_logging_session():
|
| 9 |
global _http_session
|
| 10 |
if SUPABASE_URL and SUPABASE_SERVICE_KEY:
|
|
@@ -17,29 +33,13 @@ async def init_logging_session():
|
|
| 17 |
"Prefer": "return=minimal",
|
| 18 |
}
|
| 19 |
)
|
| 20 |
-
|
| 21 |
async def close_logging_session():
|
| 22 |
global _http_session
|
| 23 |
if _http_session:
|
| 24 |
await _http_session.close()
|
| 25 |
|
| 26 |
-
try:
|
| 27 |
-
from loguru import logger as _loguru
|
| 28 |
-
_loguru.remove()
|
| 29 |
-
_loguru.add(
|
| 30 |
-
lambda msg: print(msg, end=""),
|
| 31 |
-
format="<green>{time:HH:mm:ss}</green> | <level>{level:<8}</level> | {message}",
|
| 32 |
-
level="DEBUG",
|
| 33 |
-
colorize=True,
|
| 34 |
-
)
|
| 35 |
-
_log_fn = _loguru.log
|
| 36 |
-
except ImportError:
|
| 37 |
-
import logging as _logging
|
| 38 |
-
_logging.basicConfig(level=_logging.INFO)
|
| 39 |
-
_stdlib = _logging.getLogger("vsl")
|
| 40 |
|
| 41 |
-
def _log_fn(level: str, msg: str):
|
| 42 |
-
_stdlib.log(getattr(_logging, level, 20), msg)
|
| 43 |
|
| 44 |
async def _supabase_log(level: str, event: str, data: dict) -> None:
|
| 45 |
if not _http_session:
|
|
@@ -68,6 +68,8 @@ async def _supabase_log(level: str, event: str, data: dict) -> None:
|
|
| 68 |
except Exception as exc:
|
| 69 |
_log_fn("DEBUG", f"Supabase log push skipped: {exc}")
|
| 70 |
|
|
|
|
|
|
|
| 71 |
def log(level: str, event: str, **data) -> None:
|
| 72 |
_log_fn(level.upper(), f"[{event}] {json.dumps(data, default=str)}")
|
| 73 |
try:
|
|
|
|
| 1 |
import asyncio
|
| 2 |
import json
|
|
|
|
| 3 |
from src.core.config import SUPABASE_URL, SUPABASE_SERVICE_KEY
|
| 4 |
+
from loguru import logger as _loguru
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
|
| 8 |
|
| 9 |
_http_session = None
|
| 10 |
|
| 11 |
+
#Removing the Default Loging Template
|
| 12 |
+
_loguru.remove()
|
| 13 |
+
#Adding the Custom Loging Template
|
| 14 |
+
_loguru.add(
|
| 15 |
+
lambda msg: print(msg, end=""),
|
| 16 |
+
format="<green>{time:HH:mm:ss}</green> | <level>{level:<8}</level> | {message}",
|
| 17 |
+
level="DEBUG",
|
| 18 |
+
colorize=True,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
_log_fn = _loguru.log
|
| 22 |
+
|
| 23 |
+
# A coroutine to initialize the HTTP session
|
| 24 |
async def init_logging_session():
|
| 25 |
global _http_session
|
| 26 |
if SUPABASE_URL and SUPABASE_SERVICE_KEY:
|
|
|
|
| 33 |
"Prefer": "return=minimal",
|
| 34 |
}
|
| 35 |
)
|
| 36 |
+
# Coroutine to close the HTTP session
|
| 37 |
async def close_logging_session():
|
| 38 |
global _http_session
|
| 39 |
if _http_session:
|
| 40 |
await _http_session.close()
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
|
|
|
|
|
|
| 43 |
|
| 44 |
async def _supabase_log(level: str, event: str, data: dict) -> None:
|
| 45 |
if not _http_session:
|
|
|
|
| 68 |
except Exception as exc:
|
| 69 |
_log_fn("DEBUG", f"Supabase log push skipped: {exc}")
|
| 70 |
|
| 71 |
+
|
| 72 |
+
|
| 73 |
def log(level: str, event: str, **data) -> None:
|
| 74 |
_log_fn(level.upper(), f"[{event}] {json.dumps(data, default=str)}")
|
| 75 |
try:
|