AdarshDRC commited on
Commit
5066eba
·
verified ·
1 Parent(s): 79e267a

Update src/models.py

Browse files
Files changed (1) hide show
  1. src/models.py +428 -360
src/models.py CHANGED
@@ -1,77 +1,74 @@
1
- # src/models.py — Enterprise Lens V4
2
- # ════════════════════════════════════════════════════════════════════
3
- # Face Lane : InsightFace SCRFD-10GF + ArcFace-R100 (buffalo_l)
4
- # + AdaFace IR-50 (WebFace4M) fused → 1024-D vector
5
- # • det_size=(1280,1280) — catches small/group faces
6
- # • Quality gate: det_score ≥ 0.60, face_px ≥ 40
7
- # • Multi-scale: runs detection at 2 scales, merges
8
- # • Stores one 1024-D vector PER face
9
- # • Each vector carries base64 face-crop thumbnail
10
- # • face_quality_score + face_width_px in metadata
11
- #
12
- # Object Lane: SigLIP + DINOv2 fused 1536-D (unchanged from V3)
13
- # ════════════════════════════════════════════════════════════════════
14
 
15
- import os
16
- os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
- import asyncio
19
- import base64
20
  import functools
21
- import hashlib
22
  import io
23
  import threading
 
24
  import traceback
 
25
 
26
  import cv2
27
  import numpy as np
28
  import torch
29
- import torch.nn as nn
30
  import torch.nn.functional as F
31
  from PIL import Image
32
  from transformers import AutoImageProcessor, AutoModel, AutoProcessor
33
  from ultralytics import YOLO
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
34
 
35
- # ── InsightFace ───────────────────────────────────────────────────
36
- try:
37
- import insightface
38
- from insightface.app import FaceAnalysis
39
- INSIGHTFACE_AVAILABLE = True
40
- except ImportError:
41
- INSIGHTFACE_AVAILABLE = False
42
- print("⚠️ insightface not installed — face lane disabled")
43
- print(" Run: pip install insightface onnxruntime-silicon (mac)")
44
- print(" pip install insightface onnxruntime (linux/win)")
45
-
46
- # ── AdaFace ──────────────────────────────────────────────────────
47
- # Disabled by default — enable by setting ENABLE_ADAFACE=1 env var.
48
- # When disabled: ArcFace(512) + zeros(512) = 1024-D (fully functional).
49
- ADAFACE_WEIGHTS_AVAILABLE = False # controlled by ENABLE_ADAFACE env var
50
-
51
- # ── Constants ─────────────────────────────────────────────────────
52
- YOLO_PERSON_CLASS_ID = 0
53
- MIN_FACE_SIZE = 20 # lowered: 40 missed small faces in group photos
54
- MAX_FACES_PER_IMAGE = 12 # slightly higher cap for group photos
55
- MAX_CROPS = 6 # max YOLO object crops per image
56
- MAX_IMAGE_SIZE = 640 # object lane longest edge
57
- DET_SIZE_PRIMARY = (1280, 1280) # V4: 1280 for small-face detection
58
- DET_SIZE_SECONDARY = (640, 640) # fallback / 2nd scale
59
- FACE_CROP_THUMB_SIZE = 112 # face thumbnail for Pinecone metadata
60
- FACE_CROP_QUALITY = 80 # JPEG quality for thumbnails
61
- FACE_QUALITY_GATE = 0.35 # lowered from 0.60 — accepts sunglasses, angles, smiles
62
- # Multi-scale pyramid — tried in order, results merged with IoU dedup
63
- DET_SCALES = [(1280, 1280), (960, 960), (640, 640)]
64
- IOU_DEDUP_THRESHOLD = 0.45 # suppress duplicate detections across scales
65
- FACE_DIM = 512 # ArcFace embedding dimension
66
- ADAFACE_DIM = 512 # AdaFace embedding dimension
67
- FUSED_FACE_DIM = 1024 # ArcFace + AdaFace concatenated
68
-
69
-
70
- # ════════════════════════════════════════════════════════════════
71
- # Utility functions
72
- # ════════════════════════════════════════════════════════════════
73
 
74
  def _resize_pil(img: Image.Image, max_side: int = MAX_IMAGE_SIZE) -> Image.Image:
 
 
 
 
 
 
 
 
 
 
75
  w, h = img.size
76
  if max(w, h) <= max_side:
77
  return img
@@ -79,32 +76,33 @@ def _resize_pil(img: Image.Image, max_side: int = MAX_IMAGE_SIZE) -> Image.Image
79
  return img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
80
 
81
 
82
- def _img_hash(image_path: str) -> str:
83
- h = hashlib.md5()
84
- with open(image_path, "rb") as f:
85
- h.update(f.read(65536))
86
- return h.hexdigest()
87
-
88
-
89
  def _crop_to_b64(
90
  img_bgr: np.ndarray,
91
  x1: int, y1: int, x2: int, y2: int,
92
- thumb_size: int = FACE_CROP_THUMB_SIZE,
93
  ) -> str:
94
- """Crop face from BGR image with 20% padding, return base64 JPEG thumbnail."""
95
- H, W = img_bgr.shape[:2]
96
- w, h = x2 - x1, y2 - y1
97
- pad_x = int(w * 0.20)
98
- pad_y = int(h * 0.20)
99
- cx1 = max(0, x1 - pad_x)
100
- cy1 = max(0, y1 - pad_y)
101
- cx2 = min(W, x2 + pad_x)
102
- cy2 = min(H, y2 + pad_y)
103
- crop = img_bgr[cy1:cy2, cx1:cx2]
 
 
 
 
 
 
 
 
 
104
  if crop.size == 0:
105
  return ""
106
- pil = Image.fromarray(crop[:, :, ::-1]) # BGR → RGB
107
- pil = pil.resize((thumb_size, thumb_size), Image.LANCZOS)
108
  buf = io.BytesIO()
109
  pil.save(buf, format="JPEG", quality=FACE_CROP_QUALITY)
110
  return base64.b64encode(buf.getvalue()).decode()
@@ -113,216 +111,315 @@ def _crop_to_b64(
113
  def _face_crop_for_adaface(
114
  img_bgr: np.ndarray,
115
  x1: int, y1: int, x2: int, y2: int,
116
- ) -> np.ndarray:
117
  """
118
- Crop and normalise face for AdaFace IR-50 input.
119
- Returns float32 numpy array (3, 112, 112) normalised to [-1, 1].
 
 
 
 
 
 
 
 
 
 
 
 
120
  """
121
- H, W = img_bgr.shape[:2]
122
- w, h = x2 - x1, y2 - y1
123
- pad_x = int(w * 0.10)
124
- pad_y = int(h * 0.10)
125
- cx1 = max(0, x1 - pad_x)
126
- cy1 = max(0, y1 - pad_y)
127
- cx2 = min(W, x2 + pad_x)
128
- cy2 = min(H, y2 + pad_y)
129
- crop = img_bgr[cy1:cy2, cx1:cx2]
130
  if crop.size == 0:
131
  return None
132
- rgb = crop[:, :, ::-1].copy() # BGR → RGB
133
- pil = Image.fromarray(rgb).resize((112, 112), Image.LANCZOS)
134
- arr = np.array(pil, dtype=np.float32) / 255.0
135
- arr = (arr - 0.5) / 0.5 # normalise [-1, 1]
136
- return arr.transpose(2, 0, 1) # HWC → CHW
137
-
138
 
139
 
140
  def _clahe_enhance(bgr: np.ndarray) -> np.ndarray:
141
- """CLAHE on luminance — improves detection on dark/washed/low-contrast photos."""
142
- lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
143
- l, a, b = cv2.split(lab)
144
- clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
145
- l_eq = clahe.apply(l)
146
- return cv2.cvtColor(cv2.merge([l_eq, a, b]), cv2.COLOR_LAB2BGR)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
 
148
 
149
  def _iou(box_a: list, box_b: list) -> float:
150
- """IoU between two [x1,y1,x2,y2] boxes."""
151
- xa = max(box_a[0], box_b[0]); ya = max(box_a[1], box_b[1])
152
- xb = min(box_a[2], box_b[2]); yb = min(box_a[3], box_b[3])
 
 
 
 
 
 
 
 
 
 
 
153
  inter = max(0, xb - xa) * max(0, yb - ya)
154
  if inter == 0:
155
  return 0.0
156
- area_a = (box_a[2]-box_a[0]) * (box_a[3]-box_a[1])
157
- area_b = (box_b[2]-box_b[0]) * (box_b[3]-box_b[1])
158
  return inter / (area_a + area_b - inter)
159
 
160
 
161
  def _dedup_faces(faces_list: list, iou_thresh: float = IOU_DEDUP_THRESHOLD) -> list:
162
- """Remove duplicate detections across scales/flips. Keep highest det_score."""
 
 
 
 
 
 
 
 
 
 
163
  if not faces_list:
164
  return []
165
  faces_list = sorted(faces_list, key=lambda f: float(f.det_score), reverse=True)
166
  kept = []
167
  for face in faces_list:
168
- b = face.bbox.astype(int)
169
  box = [b[0], b[1], b[2], b[3]]
170
- duplicate = any(_iou(box, [k.bbox.astype(int)[i] for i in range(4)]) > iou_thresh for k in kept)
171
- if not duplicate:
172
  kept.append(face)
173
  return kept
174
 
175
- # ════════════════════════════════════════════════════════════════
176
- # AIModelManager — V4
177
- # ════════════════════════════════════════════════════════════════
 
178
 
179
  class AIModelManager:
 
 
 
 
 
 
 
 
 
 
 
180
  def __init__(self):
181
  self.device = (
182
- "cuda" if torch.cuda.is_available()
183
- else ("mps" if torch.backends.mps.is_available() else "cpu")
 
184
  )
185
  print(f"🚀 Loading models onto: {self.device.upper()}...")
186
 
187
- # ── Object Lane: SigLIP + DINOv2 (unchanged) ─────────────
188
  print("📦 Loading SigLIP...")
189
  self.siglip_processor = AutoProcessor.from_pretrained(
190
  "google/siglip-base-patch16-224", use_fast=True)
191
- self.siglip_model = AutoModel.from_pretrained(
192
- "google/siglip-base-patch16-224").to(self.device).eval()
 
 
193
 
 
194
  print("📦 Loading DINOv2...")
195
  self.dinov2_processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
196
- self.dinov2_model = AutoModel.from_pretrained(
197
- "facebook/dinov2-base").to(self.device).eval()
 
 
198
 
 
199
  if self.device == "cuda":
200
  self.siglip_model = self.siglip_model.half()
201
  self.dinov2_model = self.dinov2_model.half()
202
 
203
- # ── YOLO for object segmentation ─────────────────────────
204
  print("📦 Loading YOLO11n-seg...")
205
  self.yolo = YOLO("yolo11n-seg.pt")
206
 
207
- # ── Face Lane: InsightFace SCRFD + ArcFace-R100 ───────────
208
- # V4: ALWAYS use buffalo_l (SCRFD-10GF + ArcFace-R100)
209
- # even on CPU — accuracy matters more than speed here.
210
- # det_size=1280 catches faces as small as ~10px in source.
211
- self.face_app = None
212
- if INSIGHTFACE_AVAILABLE:
213
- try:
214
- print("📦 Loading InsightFace buffalo_l (SCRFD-10GF + ArcFace-R100)...")
215
- self.face_app = FaceAnalysis(
216
- name="buffalo_l",
217
- providers=(
218
- ["CUDAExecutionProvider", "CPUExecutionProvider"]
219
- if self.device == "cuda"
220
- else ["CPUExecutionProvider"]
221
- ),
222
- )
223
- self.face_app.prepare(
224
- ctx_id=0 if self.device == "cuda" else -1,
225
- det_size=DET_SIZE_PRIMARY, # 1280×1280 key for small faces
226
- )
227
- # Warmup
228
- test_img = np.zeros((112, 112, 3), dtype=np.uint8)
229
- self.face_app.get(test_img)
230
- print("✅ InsightFace buffalo_l loaded — SCRFD+ArcFace face lane ACTIVE")
231
- print(f" det_size={DET_SIZE_PRIMARY} | quality_gate={FACE_QUALITY_GATE}")
232
- except Exception as e:
233
- print(f"❌ InsightFace init FAILED: {e}")
234
- print(traceback.format_exc())
235
- self.face_app = None
236
- else:
237
- print("❌ InsightFace NOT installed")
238
-
239
- # ── AdaFace IR-50 (CVPR 2022) — quality-adaptive fusion ───
240
- # Fused with ArcFace → 1024-D face vector
241
- # Weights: adaface_ir50_webface4m.ckpt from HuggingFace
242
  self.adaface_model = None
243
  self._load_adaface()
244
 
245
- # Thread safety for ONNX
246
- self._face_lock = threading.Lock()
247
- self._cache = {}
248
- self._cache_maxsize = 128
249
- adaface_status = "FULL FUSION u2705" if self.adaface_model else "ZERO-PADDED u26a0ufe0f (AdaFace weights missing)"
250
- print("")
251
- print("u2705 Enterprise Lens V4 u2014 Models Ready")
252
- print(f" Device : {self.device.upper()}")
253
- print(f" InsightFace : buffalo_l (SCRFD-10GF + ArcFace-R100)")
254
- print(f" AdaFace : {adaface_status}")
255
- print(f" Face vector dim : {FUSED_FACE_DIM} <- enterprise-faces MUST be {FUSED_FACE_DIM}-D")
256
- print(f" Object vector dim : 1536 <- enterprise-objects MUST be 1536-D")
257
- print(f" Quality gate : det_score >= {FACE_QUALITY_GATE}, face_px >= {MIN_FACE_SIZE}")
258
- print(f" Detection size : {DET_SIZE_PRIMARY}")
259
- print("")
260
 
 
261
  def _load_adaface(self):
262
  """
263
- AdaFace IR-50 MS1MV2 disabled for now.
264
- Face vectors use ArcFace(512) + zeros(512) = 1024-D.
265
- This is fully functional — cosine similarity works correctly.
266
- Re-enable by setting ENABLE_ADAFACE=1 env var when HF token
267
- injection into Docker build is confirmed working.
 
 
 
 
 
268
  """
269
- enable = os.getenv("ENABLE_ADAFACE", "0").strip() == "1"
270
- hf_token_present = bool(os.getenv("HF_TOKEN", "").strip())
271
- print(f" ENABLE_ADAFACE={os.getenv('ENABLE_ADAFACE', 'NOT SET')}")
272
- print(f" HF_TOKEN present={'YES' if hf_token_present else 'NO (not set or empty)'}")
273
- if not enable:
274
- print("⚠️ AdaFace disabled (ENABLE_ADAFACE != 1) — using ArcFace zero-padded 1024-D")
275
- self.adaface_model = None
276
  return
277
 
278
- # Full loading code kept here for when AdaFace is re-enabled
279
- import sys
280
- HF_TOKEN = os.getenv("HF_TOKEN", None)
281
  REPO_ID = "minchul/cvlface_adaface_ir50_ms1mv2"
282
  CACHE_PATH = os.path.expanduser("~/.cvlface_cache/minchul/cvlface_adaface_ir50_ms1mv2")
283
  try:
284
  from huggingface_hub import hf_hub_download
 
 
285
  print("📦 Loading AdaFace IR-50 MS1MV2...")
286
  os.makedirs(CACHE_PATH, exist_ok=True)
 
287
  hf_hub_download(repo_id=REPO_ID, filename="files.txt",
288
- token=HF_TOKEN, local_dir=CACHE_PATH, local_dir_use_symlinks=False)
 
289
  with open(os.path.join(CACHE_PATH, "files.txt")) as f:
290
  extra = [x.strip() for x in f.read().split("\n") if x.strip()]
291
  for fname in extra + ["config.json", "wrapper.py", "model.safetensors"]:
292
  fpath = os.path.join(CACHE_PATH, fname)
293
  if not os.path.exists(fpath):
294
  hf_hub_download(repo_id=REPO_ID, filename=fname,
295
- token=HF_TOKEN, local_dir=CACHE_PATH, local_dir_use_symlinks=False)
 
 
296
  cwd = os.getcwd()
297
  os.chdir(CACHE_PATH)
298
  sys.path.insert(0, CACHE_PATH)
299
  try:
300
- from transformers import AutoModel as _HF_AutoModel
301
- model = _HF_AutoModel.from_pretrained(
302
  CACHE_PATH, trust_remote_code=True, token=HF_TOKEN)
303
  finally:
304
  os.chdir(cwd)
305
- if CACHE_PATH in sys.path: sys.path.remove(CACHE_PATH)
 
 
306
  model = model.to(self.device).eval()
307
  with torch.no_grad():
308
  out = model(torch.zeros(1, 3, 112, 112).to(self.device))
309
  emb = out if isinstance(out, torch.Tensor) else out.embedding
310
- assert emb.shape[-1] == ADAFACE_DIM
 
311
  self.adaface_model = model
312
- print(f"✅ AdaFace IR-50 loaded — 1024-D FULL FUSION active")
 
313
  except Exception as e:
314
  print(f"⚠️ AdaFace load failed: {e} — falling back to zero-padded 1024-D")
315
  self.adaface_model = None
316
 
317
- # ── Object Lane: batched SigLIP + DINOv2 embedding ───────────
318
- def _embed_crops_batch(self, crops: list) -> list:
319
- """Embed a list of PIL images → list of 1536-D numpy arrays."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  if not crops:
321
  return []
322
  with torch.no_grad():
323
  # SigLIP
324
- sig_in = self.siglip_processor(images=crops, return_tensors="pt", padding=True)
325
- sig_in = {k: v.to(self.device) for k, v in sig_in.items()}
326
  if self.device == "cuda":
327
  sig_in = {k: v.half() if v.dtype == torch.float32 else v
328
  for k, v in sig_in.items()}
@@ -336,14 +433,12 @@ class AIModelManager:
336
  sig_out = sig_out.last_hidden_state[:, 0, :]
337
  elif isinstance(sig_out, tuple):
338
  sig_out = sig_out[0]
339
- # sig_out is now a tensor
340
- if not isinstance(sig_out, torch.Tensor):
341
- sig_out = sig_out[0]
342
  sig_vecs = F.normalize(sig_out.float(), p=2, dim=1).cpu()
343
 
344
- # DINOv2
345
- dino_in = self.dinov2_processor(images=crops, return_tensors="pt")
346
- dino_in = {k: v.to(self.device) for k, v in dino_in.items()}
 
347
  if self.device == "cuda":
348
  dino_in = {k: v.half() if v.dtype == torch.float32 else v
349
  for k, v in dino_in.items()}
@@ -352,114 +447,88 @@ class AIModelManager:
352
  dino_out.last_hidden_state[:, 0, :].float(), p=2, dim=1).cpu()
353
 
354
  fused = F.normalize(torch.cat([sig_vecs, dino_vecs], dim=1), p=2, dim=1)
355
- return [fused[i].numpy() for i in range(len(crops))]
356
 
357
- # ── AdaFace embedding for a single face crop ─────────────────
358
- def _adaface_embed(self, face_arr_chw: np.ndarray) -> np.ndarray:
359
- """
360
- Run AdaFace IR-50 MS1MV2 on a preprocessed (3,112,112) float32 array.
361
- Input : CHW float32, normalised to [-1, 1]
362
- Output: 512-D L2-normalised numpy embedding, or None on failure.
363
-
364
- The cvlface model may return a tensor directly or an object
365
- with an .embedding attribute — both cases handled.
366
- """
367
- if self.adaface_model is None or face_arr_chw is None:
368
- return None
369
- try:
370
- t = torch.from_numpy(face_arr_chw).unsqueeze(0) # (1,3,112,112)
371
- t = t.to(self.device)
372
- if self.device == "cuda":
373
- t = t.half()
374
- with torch.no_grad():
375
- out = self.adaface_model(t)
376
- # Handle both raw tensor and object-with-embedding outputs
377
- emb = out if isinstance(out, torch.Tensor) else out.embedding
378
- emb = F.normalize(emb.float(), p=2, dim=1)
379
- return emb[0].cpu().numpy()
380
- except Exception as e:
381
- print(f"⚠️ AdaFace inference error: {e}")
382
- return None
383
 
384
- # ── V4 Face detection + dual encoding ────────────────────────
385
- def _detect_and_encode_faces(self, img_np: np.ndarray) -> list:
386
  """
387
- Detect ALL faces using InsightFace SCRFD-10GF at 1280px.
388
- For each face:
389
- - ArcFace-R100 embedding (512-D, from InsightFace)
390
- - AdaFace IR-50 embedding (512-D, fused quality-adaptive)
391
- - Concatenate + L2-normalise → 1024-D final vector
392
- - Quality gate: det_score 0.60, face width ≥ 40px
393
- - Base64 thumbnail stored for UI
394
-
395
- Returns list of dicts with keys:
396
- type, vector (1024-D or 512-D), face_idx, bbox,
397
- face_crop, det_score, face_quality, face_width_px
 
 
 
 
 
 
 
 
 
 
 
 
398
  """
399
  if self.face_app is None:
400
- print("⚠️ face_app is None — InsightFace not loaded")
401
  return []
402
 
403
  try:
404
- # InsightFace expects BGR
405
  if img_np.dtype != np.uint8:
406
  img_np = (img_np * 255).astype(np.uint8)
407
  bgr = img_np[:, :, ::-1].copy() if img_np.shape[2] == 3 else img_np.copy()
408
 
409
- # ── Preprocessing: CLAHE contrast enhancement ─────────
410
- # Helps with dark/overexposed/low-contrast photos
411
  bgr_enhanced = _clahe_enhance(bgr)
412
 
413
- # ── Multi-scale + flip detection ──────────────────────
414
- # Run SCRFD at multiple resolutions AND on horizontally
415
- # flipped image. Catches faces that one scale/orientation misses.
416
- # Results are merged and deduplicated by IoU.
417
  all_raw_faces = []
418
  H, W = bgr.shape[:2]
419
 
420
  for scale in DET_SCALES:
421
- # Resize to this scale for detection
422
  scale_w = min(W, scale[0])
423
  scale_h = min(H, scale[1])
424
- if scale_w == W and scale_h == H:
425
- bgr_scaled = bgr_enhanced
426
- else:
427
- bgr_scaled = cv2.resize(bgr_enhanced, (scale_w, scale_h))
428
-
429
- print(f"🔍 SCRFD detection at {scale_w}×{scale_h}...")
430
- # Temporarily set det_size for this scale
431
  try:
432
  self.face_app.det_model.input_size = scale
433
  with self._face_lock:
434
  faces_at_scale = self.face_app.get(bgr_scaled)
435
- # Scale bboxes back to original dimensions
436
- sx = W / scale_w; sy = H / scale_h
437
  for f in faces_at_scale:
438
  if sx != 1.0 or sy != 1.0:
439
  f.bbox[0] *= sx; f.bbox[1] *= sy
440
  f.bbox[2] *= sx; f.bbox[3] *= sy
441
  all_raw_faces.extend(faces_at_scale)
442
  except Exception:
443
- pass # scale failed, continue
444
 
445
- # Horizontal flip pass — catches profile/turned faces
446
  bgr_flip = cv2.flip(bgr_enhanced, 1)
447
  try:
448
  self.face_app.det_model.input_size = DET_SIZE_PRIMARY
449
  with self._face_lock:
450
  faces_flip = self.face_app.get(bgr_flip)
451
- # Mirror bboxes back to original orientation
452
  for f in faces_flip:
453
  x1, y1, x2, y2 = f.bbox
454
- f.bbox[0] = W - x2; f.bbox[2] = W - x1
 
455
  all_raw_faces.extend(faces_flip)
456
  except Exception:
457
  pass
458
 
459
- # Restore primary det_size
460
  self.face_app.det_model.input_size = DET_SIZE_PRIMARY
461
 
462
- # Deduplicate across scales and flip
463
  faces = _dedup_faces(all_raw_faces)
464
  print(f" Raw detections: {len(all_raw_faces)} → after dedup: {len(faces)}")
465
 
@@ -470,7 +539,6 @@ class AIModelManager:
470
  if accepted >= MAX_FACES_PER_IMAGE:
471
  break
472
 
473
- # ── Bounding box ──────────────────────────────────
474
  bbox_raw = face.bbox.astype(int)
475
  x1, y1, x2, y2 = bbox_raw
476
  x1 = max(0, x1); y1 = max(0, y1)
@@ -479,128 +547,116 @@ class AIModelManager:
479
  if w <= 0 or h <= 0:
480
  continue
481
 
482
- # ── Quality gate 1: minimum size ──────────────────
483
  if w < MIN_FACE_SIZE or h < MIN_FACE_SIZE:
484
  print(f" Face {idx}: SKIP — too small ({w}×{h}px)")
485
  continue
486
 
487
- # ── Quality gate 2: detection confidence ──────────
488
  det_score = float(face.det_score) if hasattr(face, "det_score") else 1.0
489
  if det_score < FACE_QUALITY_GATE:
490
  print(f" Face {idx}: SKIP — low det_score ({det_score:.3f})")
491
  continue
492
 
493
- # ── ArcFace embedding (from InsightFace) ──────────
494
  if face.embedding is None:
495
  continue
 
 
496
  arcface_vec = face.embedding.astype(np.float32)
497
  n = np.linalg.norm(arcface_vec)
498
  if n > 0:
499
  arcface_vec = arcface_vec / n
500
 
501
- # ── AdaFace embedding (quality-adaptive) ──────────
502
- face_chw = _face_crop_for_adaface(bgr, x1, y1, x2, y2)
503
  adaface_vec = self._adaface_embed(face_chw)
504
 
505
- # ── Fuse: ArcFace + AdaFace → 1024-D ─────────────
506
- # ALWAYS output FUSED_FACE_DIM (1024) so Pinecone index
507
- # dimension never mismatches, regardless of AdaFace status.
508
  if adaface_vec is not None:
509
- # Full fusion: ArcFace(512) + AdaFace(512) → 1024-D
510
  fused_raw = np.concatenate([arcface_vec, adaface_vec])
511
  else:
512
- # AdaFace unavailable — pad with zeros to maintain 1024-D
513
- # The ArcFace half still carries full identity signal;
514
- # zero padding is neutral and doesn't corrupt similarity.
515
- print(" ⚠️ AdaFace unavailable — padding to 1024-D")
516
  fused_raw = np.concatenate([arcface_vec,
517
  np.zeros(ADAFACE_DIM, dtype=np.float32)])
518
- n2 = np.linalg.norm(fused_raw)
519
  final_vec = (fused_raw / n2) if n2 > 0 else fused_raw
520
- vec_dim = FUSED_FACE_DIM # always 1024
521
 
522
- # ── Face crop thumbnail for UI ─────────────────────
523
  face_crop_b64 = _crop_to_b64(bgr, x1, y1, x2, y2)
524
 
525
  results.append({
526
- "type": "face",
527
- "vector": final_vec,
528
- "vec_dim": vec_dim,
529
- "face_idx": accepted,
530
- "bbox": [int(x1), int(y1), int(w), int(h)],
531
- "face_crop": face_crop_b64,
532
- "det_score": det_score,
533
- "face_quality": det_score, # alias for metadata
534
- "face_width_px": int(w),
535
  })
536
  accepted += 1
537
- print(f" Face {idx}: ACCEPTED — {w}×{h}px | "
538
- f"det={det_score:.3f} | dim={vec_dim}")
539
 
540
  print(f"👤 {accepted} face(s) passed quality gate")
541
  return results
542
 
543
  except Exception as e:
544
- print(f"🟠 InsightFace error: {e}")
545
- print(traceback.format_exc()[-600:])
546
  return []
547
 
548
- # ── Main process_image ────────────────────────────────────────
549
  def process_image(
550
  self,
551
- image_path: str,
552
- is_query: bool = False,
553
  detect_faces: bool = True,
554
- ) -> list:
555
  """
556
- Full pipeline for one image.
557
-
558
- Returns list of vector dicts:
559
- Face: {type, vector (1024-D), face_idx, bbox, face_crop,
560
- det_score, face_quality, face_width_px}
561
- Object: {type, vector (1536-D)}
562
-
563
- V4 changes vs V3:
564
- - SCRFD at 1280px (not 640) catches small/group faces
565
- - buffalo_l always (not buffalo_sc on CPU)
566
- - ArcFace + AdaFace fused 1024-D vectors
567
- - Quality gate: det_score ≥ 0.60, width ≥ 40px
568
- - Multi-scale: detect at 1280, retry at 640 if 0 faces found
 
 
 
 
569
  """
570
- cache_key = f"{_img_hash(image_path)}_{detect_faces}_{is_query}"
571
- if cache_key in self._cache:
572
- print("⚡ Cache hit")
573
- return self._cache[cache_key]
 
 
574
 
575
  extracted = []
576
  original_pil = Image.open(image_path).convert("RGB")
577
- img_np = np.array(original_pil) # RGB uint8
578
  faces_found = False
579
 
580
- # ════════════════════════════════════════════════════════
581
- # FACE LANE
582
- # V4: Run at full resolution (up to 1280px) to catch small
583
- # faces in group photos. If 0 faces detected, retry at
584
- # the original resolution (multi-scale fallback).
585
- # ════════════════════════════════════════════════════════
586
  if detect_faces and self.face_app is not None:
587
- # Multi-scale + CLAHE + flip all handled inside _detect_and_encode_faces
588
- # Pass the full-resolution image — internal scaling handles the rest
589
  face_results = self._detect_and_encode_faces(img_np)
590
-
591
  if face_results:
592
  faces_found = True
593
- for fr in face_results:
594
- extracted.append(fr)
595
-
596
- # ════════════════════════════════════════════════════════
597
- # OBJECT LANE
598
- # Always runs even when faces are found.
599
- # PERSON-class YOLO crops are skipped when faces active
600
- # to avoid double-counting people.
601
- # ════════════════════════════════════════════════════════
602
- crops_pil = [_resize_pil(original_pil, MAX_IMAGE_SIZE)] # full image
603
- yolo_results = self.yolo(image_path, conf=0.5, verbose=False)
 
 
604
 
605
  for r in yolo_results:
606
  if r.masks is not None:
@@ -612,11 +668,10 @@ class AIModelManager:
612
  if len(polygon) < 3:
613
  continue
614
  x, y, w, h = cv2.boundingRect(polygon)
615
- if w < 30 or h < 30:
616
  continue
617
- crop = original_pil.crop((x, y, x + w, y + h))
618
- crops_pil.append(crop)
619
- if len(crops_pil) >= MAX_CROPS + 1:
620
  break
621
  elif r.boxes is not None:
622
  for box in r.boxes:
@@ -624,33 +679,46 @@ class AIModelManager:
624
  if faces_found and cls_id == YOLO_PERSON_CLASS_ID:
625
  continue
626
  x1, y1, x2, y2 = box.xyxy[0].tolist()
627
- if (x2 - x1) < 30 or (y2 - y1) < 30:
628
  continue
629
- crop = original_pil.crop((x1, y1, x2, y2))
630
- crops_pil.append(crop)
631
- if len(crops_pil) >= MAX_CROPS + 1:
632
  break
633
 
634
- crops = [_resize_pil(c, MAX_IMAGE_SIZE) for c in crops_pil]
635
- print(f"🧠 Embedding {len(crops)} object crop(s)...")
636
- obj_vecs = self._embed_crops_batch(crops)
637
- for vec in obj_vecs:
638
- extracted.append({"type": "object", "vector": vec})
 
 
 
 
 
 
 
 
 
 
 
 
639
 
640
- # Cache
641
- if len(self._cache) >= self._cache_maxsize:
642
- del self._cache[next(iter(self._cache))]
643
- self._cache[cache_key] = extracted
644
  return extracted
645
 
646
  async def process_image_async(
647
  self,
648
  image_path: str,
649
- is_query: bool = False,
650
  detect_faces: bool = True,
651
- ) -> list:
 
 
 
 
 
 
 
652
  loop = asyncio.get_event_loop()
653
  return await loop.run_in_executor(
654
  None,
655
- functools.partial(self.process_image, image_path, is_query, detect_faces),
656
  )
 
1
+ """
2
+ src/models.py — AI inference pipeline: face detection + object embedding.
 
 
 
 
 
 
 
 
 
 
 
3
 
4
+ Two independent lanes:
5
+ Face lane : InsightFace SCRFD detection → ArcFace + AdaFace → 1024-D vector
6
+ Object lane : YOLO segmentation crops → SigLIP + DINOv2 → 1536-D vector
7
+
8
+ Both lanes run on every image. main.py decides which results to use for search.
9
+
10
+ Key design decisions:
11
+ - Multi-scale + horizontal-flip detection catches small/turned faces.
12
+ - CLAHE pre-processing recovers detail in dark / over-exposed photos.
13
+ - ArcFace + AdaFace fusion: identity-discriminative + quality-adaptive.
14
+ - SigLIP + DINOv2 fusion: semantic understanding + fine-grained texture.
15
+ - Results are cached by (file_hash, detect_faces) to avoid re-inference
16
+ on duplicate uploads or repeated queries of the same image.
17
+ """
18
 
 
 
19
  import functools
 
20
  import io
21
  import threading
22
+ import asyncio
23
  import traceback
24
+ import base64
25
 
26
  import cv2
27
  import numpy as np
28
  import torch
 
29
  import torch.nn.functional as F
30
  from PIL import Image
31
  from transformers import AutoImageProcessor, AutoModel, AutoProcessor
32
  from ultralytics import YOLO
33
+ import insightface
34
+ from insightface.app import FaceAnalysis
35
+
36
+ from .config import (
37
+ # Object lane
38
+ MAX_IMAGE_SIZE, MAX_CROPS, YOLO_PERSON_CLASS_ID,
39
+ YOLO_MIN_CROP_PX, YOLO_CONF_THRESHOLD,
40
+ # Face lane — detection
41
+ DET_SIZE_PRIMARY, DET_SCALES, IOU_DEDUP_THRESHOLD,
42
+ MIN_FACE_SIZE, MAX_FACES_PER_IMAGE, FACE_QUALITY_GATE,
43
+ # Face lane — dimensions
44
+ FACE_DIM, ADAFACE_DIM, FUSED_FACE_DIM,
45
+ # Thumbnails
46
+ FACE_CROP_THUMB_SIZE, FACE_CROP_QUALITY,
47
+ FACE_CROP_PADDING, ADAFACE_CROP_PADDING,
48
+ # Cache
49
+ INFERENCE_CACHE_SIZE,
50
+ # AdaFace toggle
51
+ ENABLE_ADAFACE, HF_TOKEN,
52
+ )
53
+ from .utils import img_hash
54
+
55
 
56
+ # ════════════════════════════════════════════════════════════════════
57
+ # MODULE-LEVEL UTILITY FUNCTIONS
58
+ # Pure functions — no model state, safe to call from anywhere.
59
+ # ════════════════════════════════════════════════════════════════════
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
 
61
  def _resize_pil(img: Image.Image, max_side: int = MAX_IMAGE_SIZE) -> Image.Image:
62
+ """
63
+ Resize a PIL image so its longest side is at most `max_side` pixels,
64
+ preserving aspect ratio.
65
+
66
+ Why max-side (not fixed W×H)? Fixed dimensions squash portrait/landscape
67
+ images. Preserving aspect ratio keeps faces and objects undistorted.
68
+
69
+ Why LANCZOS? It's a windowed sinc filter that considers more surrounding
70
+ pixels than bilinear/nearest, preserving fine detail on downscale.
71
+ """
72
  w, h = img.size
73
  if max(w, h) <= max_side:
74
  return img
 
76
  return img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
77
 
78
 
 
 
 
 
 
 
 
79
  def _crop_to_b64(
80
  img_bgr: np.ndarray,
81
  x1: int, y1: int, x2: int, y2: int,
 
82
  ) -> str:
83
+ """
84
+ Crop a face from a BGR image with FACE_CROP_PADDING padding,
85
+ resize to FACE_CROP_THUMB_SIZE × FACE_CROP_THUMB_SIZE,
86
+ and return as a base64-encoded JPEG string.
87
+
88
+ The 20 % padding (vs 10 % for AdaFace) ensures the UI thumbnail
89
+ includes hair, ears, and chin context — making it visually recognisable.
90
+ The thumbnail is stored in Pinecone metadata; the frontend renders it
91
+ as data:image/jpeg;base64,... without a Cloudinary round-trip.
92
+ """
93
+ H, W = img_bgr.shape[:2]
94
+ w, h = x2 - x1, y2 - y1
95
+ pad_x = int(w * FACE_CROP_PADDING)
96
+ pad_y = int(h * FACE_CROP_PADDING)
97
+ cx1 = max(0, x1 - pad_x)
98
+ cy1 = max(0, y1 - pad_y)
99
+ cx2 = min(W, x2 + pad_x)
100
+ cy2 = min(H, y2 + pad_y)
101
+ crop = img_bgr[cy1:cy2, cx1:cx2]
102
  if crop.size == 0:
103
  return ""
104
+ pil = Image.fromarray(crop[:, :, ::-1]) # BGR → RGB
105
+ pil = pil.resize((FACE_CROP_THUMB_SIZE, FACE_CROP_THUMB_SIZE), Image.LANCZOS)
106
  buf = io.BytesIO()
107
  pil.save(buf, format="JPEG", quality=FACE_CROP_QUALITY)
108
  return base64.b64encode(buf.getvalue()).decode()
 
111
  def _face_crop_for_adaface(
112
  img_bgr: np.ndarray,
113
  x1: int, y1: int, x2: int, y2: int,
114
+ ) -> np.ndarray | None:
115
  """
116
+ Crop and preprocess a face region for AdaFace IR-50 model input.
117
+
118
+ Input contract: BGR uint8 numpy array (H, W, 3)
119
+ Output contract: float32 numpy array (3, 112, 112) normalised to [-1, 1]
120
+
121
+ Why 10 % padding (not 20 %)? AdaFace expects a tight face crop; too
122
+ much background degrades embedding quality.
123
+
124
+ Why [-1, 1] normalisation? AdaFace was trained with this range.
125
+ Feeding [0, 1] or [0, 255] produces garbage embeddings because the
126
+ model's BN/weight distributions assume [-1, 1] input statistics.
127
+
128
+ Why HWC → CHW transpose? PIL and numpy use (H, W, C); PyTorch models
129
+ expect (C, H, W). The transpose bridges this convention difference.
130
  """
131
+ H, W = img_bgr.shape[:2]
132
+ w, h = x2 - x1, y2 - y1
133
+ pad_x = int(w * ADAFACE_CROP_PADDING)
134
+ pad_y = int(h * ADAFACE_CROP_PADDING)
135
+ cx1 = max(0, x1 - pad_x)
136
+ cy1 = max(0, y1 - pad_y)
137
+ cx2 = min(W, x2 + pad_x)
138
+ cy2 = min(H, y2 + pad_y)
139
+ crop = img_bgr[cy1:cy2, cx1:cx2]
140
  if crop.size == 0:
141
  return None
142
+ rgb = crop[:, :, ::-1].copy()
143
+ pil = Image.fromarray(rgb).resize((112, 112), Image.LANCZOS)
144
+ arr = np.array(pil, dtype=np.float32) / 255.0
145
+ arr = (arr - 0.5) / 0.5 # [0,1] [-1,1]
146
+ return arr.transpose(2, 0, 1) # HWC → CHW
 
147
 
148
 
149
  def _clahe_enhance(bgr: np.ndarray) -> np.ndarray:
150
+ """
151
+ Apply CLAHE (Contrast-Limited Adaptive Histogram Equalisation) to the
152
+ luminance channel of a BGR image.
153
+
154
+ Why CLAHE? Face detection fails on dark, backlit, or washed-out photos.
155
+ CLAHE improves local contrast without globally blowing out highlights.
156
+
157
+ Why LAB colour space? The L channel is pure luminance — enhancing it
158
+ leaves the colour information (A, B channels) completely untouched,
159
+ preventing skin-tone shifts.
160
+
161
+ clipLimit=2.0 — caps per-tile histogram bin amplification to prevent
162
+ noise from being treated as real contrast.
163
+ tileGridSize=(8,8) — 8×8 tiles for local adaptation; smaller = more
164
+ aggressive local correction.
165
+ """
166
+ lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
167
+ l_ch, a_ch, b_ch = cv2.split(lab)
168
+ clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
169
+ l_eq = clahe.apply(l_ch)
170
+ return cv2.cvtColor(cv2.merge([l_eq, a_ch, b_ch]), cv2.COLOR_LAB2BGR)
171
 
172
 
173
  def _iou(box_a: list, box_b: list) -> float:
174
+ """
175
+ Intersection-over-Union between two [x1, y1, x2, y2] bounding boxes.
176
+
177
+ IoU = area(intersection) / area(union)
178
+
179
+ Used by _dedup_faces to suppress duplicate face detections across
180
+ detection scales and the horizontal-flip pass.
181
+
182
+ Returns 0.0 if boxes don't overlap.
183
+ """
184
+ xa = max(box_a[0], box_b[0])
185
+ ya = max(box_a[1], box_b[1])
186
+ xb = min(box_a[2], box_b[2])
187
+ yb = min(box_a[3], box_b[3])
188
  inter = max(0, xb - xa) * max(0, yb - ya)
189
  if inter == 0:
190
  return 0.0
191
+ area_a = (box_a[2] - box_a[0]) * (box_a[3] - box_a[1])
192
+ area_b = (box_b[2] - box_b[0]) * (box_b[3] - box_b[1])
193
  return inter / (area_a + area_b - inter)
194
 
195
 
196
  def _dedup_faces(faces_list: list, iou_thresh: float = IOU_DEDUP_THRESHOLD) -> list:
197
+ """
198
+ Non-Maximum Suppression over face detections from multiple scales/flips.
199
+
200
+ Algorithm (greedy NMS):
201
+ 1. Sort detections by det_score descending.
202
+ 2. For each face, keep it only if it doesn't overlap (IoU > iou_thresh)
203
+ with any already-kept face.
204
+
205
+ Sorting by confidence first ensures the higher-quality detection "wins"
206
+ when two boxes refer to the same physical face.
207
+ """
208
  if not faces_list:
209
  return []
210
  faces_list = sorted(faces_list, key=lambda f: float(f.det_score), reverse=True)
211
  kept = []
212
  for face in faces_list:
213
+ b = face.bbox.astype(int)
214
  box = [b[0], b[1], b[2], b[3]]
215
+ if not any(_iou(box, [k.bbox.astype(int)[i] for i in range(4)]) > iou_thresh
216
+ for k in kept):
217
  kept.append(face)
218
  return kept
219
 
220
+
221
+ # ══════════════════════════════════════════���═════════════════════════
222
+ # AIModelManager
223
+ # ════════════════════════════════════════════════════════════════════
224
 
225
  class AIModelManager:
226
+ """
227
+ Loads and manages all AI models at server startup.
228
+ Thread-safe for the face lane (via _face_lock).
229
+ Cache-safe for all lanes (via _cache_lock).
230
+
231
+ Models loaded:
232
+ Object lane: SigLIP-base-patch16-224 + DINOv2-base → 1536-D fused
233
+ Face lane: InsightFace buffalo_l (SCRFD-10GF + ArcFace-R100) +
234
+ optionally AdaFace IR-50 → 1024-D fused
235
+ """
236
+
237
  def __init__(self):
238
  self.device = (
239
+ "cuda" if torch.cuda.is_available() else
240
+ "mps" if torch.backends.mps.is_available() else
241
+ "cpu"
242
  )
243
  print(f"🚀 Loading models onto: {self.device.upper()}...")
244
 
245
+ # ── Object lane: SigLIP ──────────────────────────────────
246
  print("📦 Loading SigLIP...")
247
  self.siglip_processor = AutoProcessor.from_pretrained(
248
  "google/siglip-base-patch16-224", use_fast=True)
249
+ self.siglip_model = (
250
+ AutoModel.from_pretrained("google/siglip-base-patch16-224")
251
+ .to(self.device).eval()
252
+ )
253
 
254
+ # ── Object lane: DINOv2 ──────────────────────────────────
255
  print("📦 Loading DINOv2...")
256
  self.dinov2_processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
257
+ self.dinov2_model = (
258
+ AutoModel.from_pretrained("facebook/dinov2-base")
259
+ .to(self.device).eval()
260
+ )
261
 
262
+ # FP16 halves VRAM usage on CUDA with negligible accuracy loss at inference
263
  if self.device == "cuda":
264
  self.siglip_model = self.siglip_model.half()
265
  self.dinov2_model = self.dinov2_model.half()
266
 
267
+ # ── Object lane: YOLO segmentation ──────────────────────
268
  print("📦 Loading YOLO11n-seg...")
269
  self.yolo = YOLO("yolo11n-seg.pt")
270
 
271
+ # ── Face lane: InsightFace SCRFD + ArcFace ───────────────
272
+ # buffalo_l = SCRFD-10GF detector + ArcFace-R100 recogniser.
273
+ # Always use buffalo_l (not buffalo_sc) — accuracy matters here.
274
+ print("📦 Loading InsightFace buffalo_l (SCRFD-10GF + ArcFace-R100)...")
275
+ self.face_app = FaceAnalysis(
276
+ name="buffalo_l",
277
+ providers=(
278
+ ["CUDAExecutionProvider", "CPUExecutionProvider"]
279
+ if self.device == "cuda"
280
+ else ["CPUExecutionProvider"]
281
+ ),
282
+ )
283
+ self.face_app.prepare(
284
+ ctx_id=0 if self.device == "cuda" else -1,
285
+ det_size=DET_SIZE_PRIMARY,
286
+ )
287
+ # Warmup — pre-allocates ONNX buffers so first real call isn't slow
288
+ self.face_app.get(np.zeros((112, 112, 3), dtype=np.uint8))
289
+ print(f"✅ InsightFace loaded | det_size={DET_SIZE_PRIMARY} | gate={FACE_QUALITY_GATE}")
290
+
291
+ # ── Face lane: AdaFace (optional) ────────────────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
292
  self.adaface_model = None
293
  self._load_adaface()
294
 
295
+ # ── Thread safety ────────────────────────────────────────
296
+ # _face_lock : InsightFace ONNX runtime is NOT thread-safe
297
+ # _cache_lock : protects _cache dict from concurrent read-write-evict
298
+ self._face_lock = threading.Lock()
299
+ self._cache_lock = threading.Lock()
300
+ self._cache: dict[str, list] = {}
301
+
302
+ adaface_status = "FULL FUSION ✅" if self.adaface_model else "ZERO-PADDED ⚠️ (weights missing)"
303
+ print(
304
+ f"\n✅ Enterprise Lens V4 — Models Ready\n"
305
+ f" Device : {self.device.upper()}\n"
306
+ f" Face vectors : {FUSED_FACE_DIM}-D ({adaface_status})\n"
307
+ f" Object vectors: 1536-D (SigLIP+DINOv2)\n"
308
+ f" Quality gate : det_score ≥ {FACE_QUALITY_GATE}, face_px ≥ {MIN_FACE_SIZE}\n"
309
+ )
310
 
311
+ # ── AdaFace loader ───────────────────────────────────────────────
312
  def _load_adaface(self):
313
  """
314
+ Load AdaFace IR-50 MS1MV2 from HuggingFace.
315
+ Controlled by ENABLE_ADAFACE env var (default off).
316
+
317
+ When disabled: ArcFace(512) + zeros(512) 1024-D output.
318
+ Zero-padding is cosine-neutral the ArcFace half still carries
319
+ full identity signal; padded zeros don't pull any direction.
320
+
321
+ When enabled: ArcFace(512) + AdaFace(512) → 1024-D.
322
+ AdaFace is quality-adaptive: blurry/low-quality face crops receive
323
+ downweighted embeddings, improving retrieval precision.
324
  """
325
+ if not ENABLE_ADAFACE:
326
+ print("⚠️ AdaFace disabled (ENABLE_ADAFACE != 1) — using zero-padded 1024-D")
 
 
 
 
 
327
  return
328
 
329
+ import os, sys
 
 
330
  REPO_ID = "minchul/cvlface_adaface_ir50_ms1mv2"
331
  CACHE_PATH = os.path.expanduser("~/.cvlface_cache/minchul/cvlface_adaface_ir50_ms1mv2")
332
  try:
333
  from huggingface_hub import hf_hub_download
334
+ from transformers import AutoModel as _HFAutoModel
335
+
336
  print("📦 Loading AdaFace IR-50 MS1MV2...")
337
  os.makedirs(CACHE_PATH, exist_ok=True)
338
+
339
  hf_hub_download(repo_id=REPO_ID, filename="files.txt",
340
+ token=HF_TOKEN, local_dir=CACHE_PATH,
341
+ local_dir_use_symlinks=False)
342
  with open(os.path.join(CACHE_PATH, "files.txt")) as f:
343
  extra = [x.strip() for x in f.read().split("\n") if x.strip()]
344
  for fname in extra + ["config.json", "wrapper.py", "model.safetensors"]:
345
  fpath = os.path.join(CACHE_PATH, fname)
346
  if not os.path.exists(fpath):
347
  hf_hub_download(repo_id=REPO_ID, filename=fname,
348
+ token=HF_TOKEN, local_dir=CACHE_PATH,
349
+ local_dir_use_symlinks=False)
350
+
351
  cwd = os.getcwd()
352
  os.chdir(CACHE_PATH)
353
  sys.path.insert(0, CACHE_PATH)
354
  try:
355
+ model = _HFAutoModel.from_pretrained(
 
356
  CACHE_PATH, trust_remote_code=True, token=HF_TOKEN)
357
  finally:
358
  os.chdir(cwd)
359
+ if CACHE_PATH in sys.path:
360
+ sys.path.remove(CACHE_PATH)
361
+
362
  model = model.to(self.device).eval()
363
  with torch.no_grad():
364
  out = model(torch.zeros(1, 3, 112, 112).to(self.device))
365
  emb = out if isinstance(out, torch.Tensor) else out.embedding
366
+ assert emb.shape[-1] == ADAFACE_DIM, f"Expected {ADAFACE_DIM}-D, got {emb.shape[-1]}"
367
+
368
  self.adaface_model = model
369
+ print("✅ AdaFace IR-50 loaded — 1024-D FULL FUSION active")
370
+
371
  except Exception as e:
372
  print(f"⚠️ AdaFace load failed: {e} — falling back to zero-padded 1024-D")
373
  self.adaface_model = None
374
 
375
+ # ── AdaFace inference ────────────────────────────────────────────
376
+ def _adaface_embed(self, face_arr_chw: np.ndarray | None) -> np.ndarray | None:
377
+ """
378
+ Run AdaFace on a preprocessed (3, 112, 112) float32 CHW array.
379
+ Returns a 512-D L2-normalised numpy embedding, or None on failure.
380
+
381
+ The cvlface model may return a raw tensor or an object with .embedding —
382
+ both output formats are handled here.
383
+ """
384
+ if self.adaface_model is None or face_arr_chw is None:
385
+ return None
386
+ try:
387
+ t = torch.from_numpy(face_arr_chw).unsqueeze(0).to(self.device)
388
+ if self.device == "cuda":
389
+ t = t.half()
390
+ with torch.no_grad():
391
+ out = self.adaface_model(t)
392
+ emb = out if isinstance(out, torch.Tensor) else out.embedding
393
+ return F.normalize(emb.float(), p=2, dim=1)[0].cpu().numpy()
394
+ except Exception as e:
395
+ print(f"⚠️ AdaFace inference error: {e}")
396
+ return None
397
+
398
+ # ── Object lane: batched embedding ──────────────────────────────
399
+ def _embed_crops_batch(self, crops: list[Image.Image]) -> list[np.ndarray]:
400
+ """
401
+ Embed a batch of PIL images through SigLIP and DINOv2, fuse results.
402
+
403
+ SigLIP captures semantic/language-aligned meaning ("a red sports car").
404
+ DINOv2 captures fine-grained visual texture and structure (self-supervised).
405
+ Fusing both gives vectors that are sensitive to BOTH what something IS
406
+ and what it LOOKS LIKE — better retrieval than either model alone.
407
+
408
+ Why batch? GPUs process many inputs in parallel almost as fast as one.
409
+ Why torch.no_grad()? Skips gradient graph construction — ~30 % faster,
410
+ significant memory saving at inference time.
411
+ Why F.normalize (L2)? Projects embeddings onto unit sphere.
412
+ On the unit sphere: cosine_similarity = dot_product
413
+ (cheaper and numerically stable).
414
+ Also ensures neither SigLIP nor DINOv2 dominates
415
+ the fused vector due to scale differences.
416
+ """
417
  if not crops:
418
  return []
419
  with torch.no_grad():
420
  # SigLIP
421
+ sig_in = self.siglip_processor(images=crops, return_tensors="pt", padding=True)
422
+ sig_in = {k: v.to(self.device) for k, v in sig_in.items()}
423
  if self.device == "cuda":
424
  sig_in = {k: v.half() if v.dtype == torch.float32 else v
425
  for k, v in sig_in.items()}
 
433
  sig_out = sig_out.last_hidden_state[:, 0, :]
434
  elif isinstance(sig_out, tuple):
435
  sig_out = sig_out[0]
 
 
 
436
  sig_vecs = F.normalize(sig_out.float(), p=2, dim=1).cpu()
437
 
438
+ # DINOv2 — [:, 0, :] extracts the [CLS] token which aggregates
439
+ # the global image representation across the entire sequence
440
+ dino_in = self.dinov2_processor(images=crops, return_tensors="pt")
441
+ dino_in = {k: v.to(self.device) for k, v in dino_in.items()}
442
  if self.device == "cuda":
443
  dino_in = {k: v.half() if v.dtype == torch.float32 else v
444
  for k, v in dino_in.items()}
 
447
  dino_out.last_hidden_state[:, 0, :].float(), p=2, dim=1).cpu()
448
 
449
  fused = F.normalize(torch.cat([sig_vecs, dino_vecs], dim=1), p=2, dim=1)
 
450
 
451
+ return [fused[i].numpy() for i in range(len(crops))]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
452
 
453
+ # ── Face lane: detection + dual encoding ────────────────────────
454
+ def _detect_and_encode_faces(self, img_np: np.ndarray) -> list[dict]:
455
  """
456
+ Detect all faces using InsightFace SCRFD-10GF at multiple scales,
457
+ encode each face with ArcFace-R100 + AdaFace IR-50, and return
458
+ 1024-D fused vectors.
459
+
460
+ Pipeline per face:
461
+ 1. ArcFace-R100 (512-D) from InsightFace's built-in recognition
462
+ 2. AdaFace IR-50 (512-D) from separately loaded model
463
+ 3. Concatenate + L2-normalise → 1024-D final vector
464
+ 4. Quality gates: det_score FACE_QUALITY_GATE, width ≥ MIN_FACE_SIZE
465
+
466
+ Multi-scale strategy:
467
+ - Run SCRFD at 1280, 960, and 640 px.
468
+ - Run once more on horizontally flipped image (catches turned faces).
469
+ - Merge all detections and deduplicate by IoU.
470
+ Rationale: a face that's 15 px at 640 becomes 30 px at 1280;
471
+ the detector finds it at the larger scale.
472
+
473
+ AdaFace unavailable:
474
+ Zero-pad to maintain 1024-D. The ArcFace half carries full identity
475
+ signal; zero padding is cosine-neutral (no direction bias).
476
+
477
+ Returns list of dicts:
478
+ { type, vector (1024-D), face_idx, bbox, face_crop, det_score, face_width_px }
479
  """
480
  if self.face_app is None:
 
481
  return []
482
 
483
  try:
 
484
  if img_np.dtype != np.uint8:
485
  img_np = (img_np * 255).astype(np.uint8)
486
  bgr = img_np[:, :, ::-1].copy() if img_np.shape[2] == 3 else img_np.copy()
487
 
488
+ # CLAHE: boost contrast on dark/backlit/low-contrast photos
 
489
  bgr_enhanced = _clahe_enhance(bgr)
490
 
491
+ # Multi-scale detection bboxes are scaled back to original coords
 
 
 
492
  all_raw_faces = []
493
  H, W = bgr.shape[:2]
494
 
495
  for scale in DET_SCALES:
 
496
  scale_w = min(W, scale[0])
497
  scale_h = min(H, scale[1])
498
+ bgr_scaled = (
499
+ bgr_enhanced if scale_w == W and scale_h == H
500
+ else cv2.resize(bgr_enhanced, (scale_w, scale_h))
501
+ )
 
 
 
502
  try:
503
  self.face_app.det_model.input_size = scale
504
  with self._face_lock:
505
  faces_at_scale = self.face_app.get(bgr_scaled)
506
+ sx, sy = W / scale_w, H / scale_h
 
507
  for f in faces_at_scale:
508
  if sx != 1.0 or sy != 1.0:
509
  f.bbox[0] *= sx; f.bbox[1] *= sy
510
  f.bbox[2] *= sx; f.bbox[3] *= sy
511
  all_raw_faces.extend(faces_at_scale)
512
  except Exception:
513
+ pass
514
 
515
+ # Horizontal-flip pass — catches profile/turned faces
516
  bgr_flip = cv2.flip(bgr_enhanced, 1)
517
  try:
518
  self.face_app.det_model.input_size = DET_SIZE_PRIMARY
519
  with self._face_lock:
520
  faces_flip = self.face_app.get(bgr_flip)
 
521
  for f in faces_flip:
522
  x1, y1, x2, y2 = f.bbox
523
+ f.bbox[0] = W - x2
524
+ f.bbox[2] = W - x1
525
  all_raw_faces.extend(faces_flip)
526
  except Exception:
527
  pass
528
 
529
+ # Restore primary size
530
  self.face_app.det_model.input_size = DET_SIZE_PRIMARY
531
 
 
532
  faces = _dedup_faces(all_raw_faces)
533
  print(f" Raw detections: {len(all_raw_faces)} → after dedup: {len(faces)}")
534
 
 
539
  if accepted >= MAX_FACES_PER_IMAGE:
540
  break
541
 
 
542
  bbox_raw = face.bbox.astype(int)
543
  x1, y1, x2, y2 = bbox_raw
544
  x1 = max(0, x1); y1 = max(0, y1)
 
547
  if w <= 0 or h <= 0:
548
  continue
549
 
550
+ # Quality gate 1: minimum pixel size
551
  if w < MIN_FACE_SIZE or h < MIN_FACE_SIZE:
552
  print(f" Face {idx}: SKIP — too small ({w}×{h}px)")
553
  continue
554
 
555
+ # Quality gate 2: detector confidence
556
  det_score = float(face.det_score) if hasattr(face, "det_score") else 1.0
557
  if det_score < FACE_QUALITY_GATE:
558
  print(f" Face {idx}: SKIP — low det_score ({det_score:.3f})")
559
  continue
560
 
 
561
  if face.embedding is None:
562
  continue
563
+
564
+ # ArcFace embedding (built into InsightFace buffalo_l)
565
  arcface_vec = face.embedding.astype(np.float32)
566
  n = np.linalg.norm(arcface_vec)
567
  if n > 0:
568
  arcface_vec = arcface_vec / n
569
 
570
+ # AdaFace embedding (quality-adaptive)
571
+ face_chw = _face_crop_for_adaface(bgr, x1, y1, x2, y2)
572
  adaface_vec = self._adaface_embed(face_chw)
573
 
574
+ # Fuse to 1024-D — always output FUSED_FACE_DIM regardless of AdaFace status
 
 
575
  if adaface_vec is not None:
 
576
  fused_raw = np.concatenate([arcface_vec, adaface_vec])
577
  else:
 
 
 
 
578
  fused_raw = np.concatenate([arcface_vec,
579
  np.zeros(ADAFACE_DIM, dtype=np.float32)])
580
+ n2 = np.linalg.norm(fused_raw)
581
  final_vec = (fused_raw / n2) if n2 > 0 else fused_raw
 
582
 
 
583
  face_crop_b64 = _crop_to_b64(bgr, x1, y1, x2, y2)
584
 
585
  results.append({
586
+ "type": "face",
587
+ "vector": final_vec,
588
+ "face_idx": accepted,
589
+ # bbox exposed so the frontend can draw boxes on the query image
590
+ "bbox": [int(x1), int(y1), int(w), int(h)],
591
+ "face_crop": face_crop_b64,
592
+ "det_score": det_score,
593
+ "face_width_px": int(w),
 
594
  })
595
  accepted += 1
596
+ print(f" Face {idx}: ACCEPTED — {w}×{h}px | det={det_score:.3f}")
 
597
 
598
  print(f"👤 {accepted} face(s) passed quality gate")
599
  return results
600
 
601
  except Exception as e:
602
+ print(f"🟠 InsightFace error: {e}\n{traceback.format_exc()[-600:]}")
 
603
  return []
604
 
605
+ # ── Main pipeline ────────────────────────────────────────────────
606
  def process_image(
607
  self,
608
+ image_path: str,
 
609
  detect_faces: bool = True,
610
+ ) -> list[dict]:
611
  """
612
+ Full inference pipeline for a single image.
613
+
614
+ Always runs both lanes:
615
+ Face → list of { type:"face", vector(1024-D), face_idx, bbox,
616
+ face_crop, det_score, face_width_px }
617
+ Object → list of { type:"object", vector(1536-D) }
618
+
619
+ main.py decides which lane's results to use for Pinecone operations
620
+ based on the endpoint context (upload stores both; search can use both).
621
+
622
+ Cache strategy:
623
+ Key = (md5_of_first_64KB, detect_faces)
624
+ Hit → return cached result immediately (skips all model inference)
625
+ Miss → run pipeline, cache result, evict LRU entry if over capacity
626
+
627
+ Cache is protected by _cache_lock (threading.Lock) to prevent race
628
+ conditions when MAX_CONCURRENT_INFERENCES > 1.
629
  """
630
+ cache_key = f"{img_hash(image_path)}_{detect_faces}"
631
+
632
+ with self._cache_lock:
633
+ if cache_key in self._cache:
634
+ print("⚡ Cache hit")
635
+ return self._cache[cache_key]
636
 
637
  extracted = []
638
  original_pil = Image.open(image_path).convert("RGB")
639
+ img_np = np.array(original_pil) # RGB uint8, full resolution
640
  faces_found = False
641
 
642
+ # ── Face lane ────────────────────────────────────────────
 
 
 
 
 
643
  if detect_faces and self.face_app is not None:
 
 
644
  face_results = self._detect_and_encode_faces(img_np)
 
645
  if face_results:
646
  faces_found = True
647
+ extracted.extend(face_results)
648
+
649
+ # ── Object lane ──────────────────────────────────────────
650
+ # Always runs, even when faces are found.
651
+ # Person-class YOLO crops are skipped when face lane is active
652
+ # to avoid embedding the same person twice.
653
+ #
654
+ # Crop 0 is always the full (resized) image — ensures we always
655
+ # have at least one embedding even if YOLO finds nothing.
656
+ # YOLO is given the already-loaded PIL image to avoid re-reading
657
+ # the file from disk.
658
+ crops: list[Image.Image] = []
659
+ yolo_results = self.yolo(original_pil, conf=YOLO_CONF_THRESHOLD, verbose=False)
660
 
661
  for r in yolo_results:
662
  if r.masks is not None:
 
668
  if len(polygon) < 3:
669
  continue
670
  x, y, w, h = cv2.boundingRect(polygon)
671
+ if w < YOLO_MIN_CROP_PX or h < YOLO_MIN_CROP_PX:
672
  continue
673
+ crops.append(original_pil.crop((x, y, x + w, y + h)))
674
+ if len(crops) >= MAX_CROPS:
 
675
  break
676
  elif r.boxes is not None:
677
  for box in r.boxes:
 
679
  if faces_found and cls_id == YOLO_PERSON_CLASS_ID:
680
  continue
681
  x1, y1, x2, y2 = box.xyxy[0].tolist()
682
+ if (x2 - x1) < YOLO_MIN_CROP_PX or (y2 - y1) < YOLO_MIN_CROP_PX:
683
  continue
684
+ crops.append(original_pil.crop((x1, y1, x2, y2)))
685
+ if len(crops) >= MAX_CROPS:
 
686
  break
687
 
688
+ # Prepend the full image as crop 0, then resize ALL crops uniformly.
689
+ # (Previously the full image was pre-resized before appending, causing
690
+ # _resize_pil to be called on it twice. Now we resize everything once.)
691
+ all_crops = [original_pil] + crops
692
+ all_crops = [_resize_pil(c, MAX_IMAGE_SIZE) for c in all_crops]
693
+
694
+ print(f"🧠 Embedding {len(all_crops)} object crop(s)...")
695
+ obj_vecs = self._embed_crops_batch(all_crops)
696
+ extracted.extend({"type": "object", "vector": v} for v in obj_vecs)
697
+
698
+ # Cache with lock — prevents concurrent writes from corrupting eviction
699
+ with self._cache_lock:
700
+ if len(self._cache) >= INFERENCE_CACHE_SIZE:
701
+ # Evict LRU entry (first inserted key in plain dict = oldest)
702
+ oldest = next(iter(self._cache))
703
+ del self._cache[oldest]
704
+ self._cache[cache_key] = extracted
705
 
 
 
 
 
706
  return extracted
707
 
708
  async def process_image_async(
709
  self,
710
  image_path: str,
 
711
  detect_faces: bool = True,
712
+ ) -> list[dict]:
713
+ """
714
+ Async wrapper for process_image — offloads blocking inference to a
715
+ thread-pool executor so FastAPI's event loop remains responsive.
716
+
717
+ functools.partial is used instead of a lambda to make the call
718
+ picklable, which some executor backends require.
719
+ """
720
  loop = asyncio.get_event_loop()
721
  return await loop.run_in_executor(
722
  None,
723
+ functools.partial(self.process_image, image_path, detect_faces),
724
  )