AdarshDRC commited on
Commit
59be0e4
·
verified ·
1 Parent(s): 8c901de

Delete src/models.py

Browse files
Files changed (1) hide show
  1. src/models.py +0 -724
src/models.py DELETED
@@ -1,724 +0,0 @@
1
- """
2
- src/models.py — AI inference pipeline: face detection + object embedding.
3
-
4
- Two independent lanes:
5
- Face lane : InsightFace SCRFD detection → ArcFace + AdaFace → 1024-D vector
6
- Object lane : YOLO segmentation crops → SigLIP + DINOv2 → 1536-D vector
7
-
8
- Both lanes run on every image. main.py decides which results to use for search.
9
-
10
- Key design decisions:
11
- - Multi-scale + horizontal-flip detection catches small/turned faces.
12
- - CLAHE pre-processing recovers detail in dark / over-exposed photos.
13
- - ArcFace + AdaFace fusion: identity-discriminative + quality-adaptive.
14
- - SigLIP + DINOv2 fusion: semantic understanding + fine-grained texture.
15
- - Results are cached by (file_hash, detect_faces) to avoid re-inference
16
- on duplicate uploads or repeated queries of the same image.
17
- """
18
-
19
- import functools
20
- import io
21
- import threading
22
- import asyncio
23
- import traceback
24
- import base64
25
-
26
- import cv2
27
- import numpy as np
28
- import torch
29
- import torch.nn.functional as F
30
- from PIL import Image
31
- from transformers import AutoImageProcessor, AutoModel, AutoProcessor
32
- from ultralytics import YOLO
33
- import insightface
34
- from insightface.app import FaceAnalysis
35
-
36
- from .config import (
37
- # Object lane
38
- MAX_IMAGE_SIZE, MAX_CROPS, YOLO_PERSON_CLASS_ID,
39
- YOLO_MIN_CROP_PX, YOLO_CONF_THRESHOLD,
40
- # Face lane — detection
41
- DET_SIZE_PRIMARY, DET_SCALES, IOU_DEDUP_THRESHOLD,
42
- MIN_FACE_SIZE, MAX_FACES_PER_IMAGE, FACE_QUALITY_GATE,
43
- # Face lane — dimensions
44
- FACE_DIM, ADAFACE_DIM, FUSED_FACE_DIM,
45
- # Thumbnails
46
- FACE_CROP_THUMB_SIZE, FACE_CROP_QUALITY,
47
- FACE_CROP_PADDING, ADAFACE_CROP_PADDING,
48
- # Cache
49
- INFERENCE_CACHE_SIZE,
50
- # AdaFace toggle
51
- ENABLE_ADAFACE, HF_TOKEN,
52
- )
53
- from .utils import img_hash
54
-
55
-
56
- # ════════════════════════════════════════════════════════════════════
57
- # MODULE-LEVEL UTILITY FUNCTIONS
58
- # Pure functions — no model state, safe to call from anywhere.
59
- # ════════════════════════════════════════════════════════════════════
60
-
61
- def _resize_pil(img: Image.Image, max_side: int = MAX_IMAGE_SIZE) -> Image.Image:
62
- """
63
- Resize a PIL image so its longest side is at most `max_side` pixels,
64
- preserving aspect ratio.
65
-
66
- Why max-side (not fixed W×H)? Fixed dimensions squash portrait/landscape
67
- images. Preserving aspect ratio keeps faces and objects undistorted.
68
-
69
- Why LANCZOS? It's a windowed sinc filter that considers more surrounding
70
- pixels than bilinear/nearest, preserving fine detail on downscale.
71
- """
72
- w, h = img.size
73
- if max(w, h) <= max_side:
74
- return img
75
- scale = max_side / max(w, h)
76
- return img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
77
-
78
-
79
- def _crop_to_b64(
80
- img_bgr: np.ndarray,
81
- x1: int, y1: int, x2: int, y2: int,
82
- ) -> str:
83
- """
84
- Crop a face from a BGR image with FACE_CROP_PADDING padding,
85
- resize to FACE_CROP_THUMB_SIZE × FACE_CROP_THUMB_SIZE,
86
- and return as a base64-encoded JPEG string.
87
-
88
- The 20 % padding (vs 10 % for AdaFace) ensures the UI thumbnail
89
- includes hair, ears, and chin context — making it visually recognisable.
90
- The thumbnail is stored in Pinecone metadata; the frontend renders it
91
- as data:image/jpeg;base64,... without a Cloudinary round-trip.
92
- """
93
- H, W = img_bgr.shape[:2]
94
- w, h = x2 - x1, y2 - y1
95
- pad_x = int(w * FACE_CROP_PADDING)
96
- pad_y = int(h * FACE_CROP_PADDING)
97
- cx1 = max(0, x1 - pad_x)
98
- cy1 = max(0, y1 - pad_y)
99
- cx2 = min(W, x2 + pad_x)
100
- cy2 = min(H, y2 + pad_y)
101
- crop = img_bgr[cy1:cy2, cx1:cx2]
102
- if crop.size == 0:
103
- return ""
104
- pil = Image.fromarray(crop[:, :, ::-1]) # BGR → RGB
105
- pil = pil.resize((FACE_CROP_THUMB_SIZE, FACE_CROP_THUMB_SIZE), Image.LANCZOS)
106
- buf = io.BytesIO()
107
- pil.save(buf, format="JPEG", quality=FACE_CROP_QUALITY)
108
- return base64.b64encode(buf.getvalue()).decode()
109
-
110
-
111
- def _face_crop_for_adaface(
112
- img_bgr: np.ndarray,
113
- x1: int, y1: int, x2: int, y2: int,
114
- ) -> np.ndarray | None:
115
- """
116
- Crop and preprocess a face region for AdaFace IR-50 model input.
117
-
118
- Input contract: BGR uint8 numpy array (H, W, 3)
119
- Output contract: float32 numpy array (3, 112, 112) normalised to [-1, 1]
120
-
121
- Why 10 % padding (not 20 %)? AdaFace expects a tight face crop; too
122
- much background degrades embedding quality.
123
-
124
- Why [-1, 1] normalisation? AdaFace was trained with this range.
125
- Feeding [0, 1] or [0, 255] produces garbage embeddings because the
126
- model's BN/weight distributions assume [-1, 1] input statistics.
127
-
128
- Why HWC → CHW transpose? PIL and numpy use (H, W, C); PyTorch models
129
- expect (C, H, W). The transpose bridges this convention difference.
130
- """
131
- H, W = img_bgr.shape[:2]
132
- w, h = x2 - x1, y2 - y1
133
- pad_x = int(w * ADAFACE_CROP_PADDING)
134
- pad_y = int(h * ADAFACE_CROP_PADDING)
135
- cx1 = max(0, x1 - pad_x)
136
- cy1 = max(0, y1 - pad_y)
137
- cx2 = min(W, x2 + pad_x)
138
- cy2 = min(H, y2 + pad_y)
139
- crop = img_bgr[cy1:cy2, cx1:cx2]
140
- if crop.size == 0:
141
- return None
142
- rgb = crop[:, :, ::-1].copy()
143
- pil = Image.fromarray(rgb).resize((112, 112), Image.LANCZOS)
144
- arr = np.array(pil, dtype=np.float32) / 255.0
145
- arr = (arr - 0.5) / 0.5 # [0,1] → [-1,1]
146
- return arr.transpose(2, 0, 1) # HWC → CHW
147
-
148
-
149
- def _clahe_enhance(bgr: np.ndarray) -> np.ndarray:
150
- """
151
- Apply CLAHE (Contrast-Limited Adaptive Histogram Equalisation) to the
152
- luminance channel of a BGR image.
153
-
154
- Why CLAHE? Face detection fails on dark, backlit, or washed-out photos.
155
- CLAHE improves local contrast without globally blowing out highlights.
156
-
157
- Why LAB colour space? The L channel is pure luminance — enhancing it
158
- leaves the colour information (A, B channels) completely untouched,
159
- preventing skin-tone shifts.
160
-
161
- clipLimit=2.0 — caps per-tile histogram bin amplification to prevent
162
- noise from being treated as real contrast.
163
- tileGridSize=(8,8) — 8×8 tiles for local adaptation; smaller = more
164
- aggressive local correction.
165
- """
166
- lab = cv2.cvtColor(bgr, cv2.COLOR_BGR2LAB)
167
- l_ch, a_ch, b_ch = cv2.split(lab)
168
- clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
169
- l_eq = clahe.apply(l_ch)
170
- return cv2.cvtColor(cv2.merge([l_eq, a_ch, b_ch]), cv2.COLOR_LAB2BGR)
171
-
172
-
173
- def _iou(box_a: list, box_b: list) -> float:
174
- """
175
- Intersection-over-Union between two [x1, y1, x2, y2] bounding boxes.
176
-
177
- IoU = area(intersection) / area(union)
178
-
179
- Used by _dedup_faces to suppress duplicate face detections across
180
- detection scales and the horizontal-flip pass.
181
-
182
- Returns 0.0 if boxes don't overlap.
183
- """
184
- xa = max(box_a[0], box_b[0])
185
- ya = max(box_a[1], box_b[1])
186
- xb = min(box_a[2], box_b[2])
187
- yb = min(box_a[3], box_b[3])
188
- inter = max(0, xb - xa) * max(0, yb - ya)
189
- if inter == 0:
190
- return 0.0
191
- area_a = (box_a[2] - box_a[0]) * (box_a[3] - box_a[1])
192
- area_b = (box_b[2] - box_b[0]) * (box_b[3] - box_b[1])
193
- return inter / (area_a + area_b - inter)
194
-
195
-
196
- def _dedup_faces(faces_list: list, iou_thresh: float = IOU_DEDUP_THRESHOLD) -> list:
197
- """
198
- Non-Maximum Suppression over face detections from multiple scales/flips.
199
-
200
- Algorithm (greedy NMS):
201
- 1. Sort detections by det_score descending.
202
- 2. For each face, keep it only if it doesn't overlap (IoU > iou_thresh)
203
- with any already-kept face.
204
-
205
- Sorting by confidence first ensures the higher-quality detection "wins"
206
- when two boxes refer to the same physical face.
207
- """
208
- if not faces_list:
209
- return []
210
- faces_list = sorted(faces_list, key=lambda f: float(f.det_score), reverse=True)
211
- kept = []
212
- for face in faces_list:
213
- b = face.bbox.astype(int)
214
- box = [b[0], b[1], b[2], b[3]]
215
- if not any(_iou(box, [k.bbox.astype(int)[i] for i in range(4)]) > iou_thresh
216
- for k in kept):
217
- kept.append(face)
218
- return kept
219
-
220
-
221
- # ════════════════════════════════════════════════════════════════════
222
- # AIModelManager
223
- # ════════════════════════════════════════════════════════════════════
224
-
225
- class AIModelManager:
226
- """
227
- Loads and manages all AI models at server startup.
228
- Thread-safe for the face lane (via _face_lock).
229
- Cache-safe for all lanes (via _cache_lock).
230
-
231
- Models loaded:
232
- Object lane: SigLIP-base-patch16-224 + DINOv2-base → 1536-D fused
233
- Face lane: InsightFace buffalo_l (SCRFD-10GF + ArcFace-R100) +
234
- optionally AdaFace IR-50 → 1024-D fused
235
- """
236
-
237
- def __init__(self):
238
- self.device = (
239
- "cuda" if torch.cuda.is_available() else
240
- "mps" if torch.backends.mps.is_available() else
241
- "cpu"
242
- )
243
- print(f"🚀 Loading models onto: {self.device.upper()}...")
244
-
245
- # ── Object lane: SigLIP ──────────────────────────────────
246
- print("📦 Loading SigLIP...")
247
- self.siglip_processor = AutoProcessor.from_pretrained(
248
- "google/siglip-base-patch16-224", use_fast=True)
249
- self.siglip_model = (
250
- AutoModel.from_pretrained("google/siglip-base-patch16-224")
251
- .to(self.device).eval()
252
- )
253
-
254
- # ── Object lane: DINOv2 ──────────────────────────────────
255
- print("📦 Loading DINOv2...")
256
- self.dinov2_processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
257
- self.dinov2_model = (
258
- AutoModel.from_pretrained("facebook/dinov2-base")
259
- .to(self.device).eval()
260
- )
261
-
262
- # FP16 halves VRAM usage on CUDA with negligible accuracy loss at inference
263
- if self.device == "cuda":
264
- self.siglip_model = self.siglip_model.half()
265
- self.dinov2_model = self.dinov2_model.half()
266
-
267
- # ── Object lane: YOLO segmentation ──────────────────────
268
- print("📦 Loading YOLO11n-seg...")
269
- self.yolo = YOLO("yolo11n-seg.pt")
270
-
271
- # ── Face lane: InsightFace SCRFD + ArcFace ───────────────
272
- # buffalo_l = SCRFD-10GF detector + ArcFace-R100 recogniser.
273
- # Always use buffalo_l (not buffalo_sc) — accuracy matters here.
274
- print("📦 Loading InsightFace buffalo_l (SCRFD-10GF + ArcFace-R100)...")
275
- self.face_app = FaceAnalysis(
276
- name="buffalo_l",
277
- providers=(
278
- ["CUDAExecutionProvider", "CPUExecutionProvider"]
279
- if self.device == "cuda"
280
- else ["CPUExecutionProvider"]
281
- ),
282
- )
283
- self.face_app.prepare(
284
- ctx_id=0 if self.device == "cuda" else -1,
285
- det_size=DET_SIZE_PRIMARY,
286
- )
287
- # Warmup — pre-allocates ONNX buffers so first real call isn't slow
288
- self.face_app.get(np.zeros((112, 112, 3), dtype=np.uint8))
289
- print(f"✅ InsightFace loaded | det_size={DET_SIZE_PRIMARY} | gate={FACE_QUALITY_GATE}")
290
-
291
- # ── Face lane: AdaFace (optional) ────────────────────────
292
- self.adaface_model = None
293
- self._load_adaface()
294
-
295
- # ── Thread safety ────────────────────────────────────────
296
- # _face_lock : InsightFace ONNX runtime is NOT thread-safe
297
- # _cache_lock : protects _cache dict from concurrent read-write-evict
298
- self._face_lock = threading.Lock()
299
- self._cache_lock = threading.Lock()
300
- self._cache: dict[str, list] = {}
301
-
302
- adaface_status = "FULL FUSION ✅" if self.adaface_model else "ZERO-PADDED ⚠️ (weights missing)"
303
- print(
304
- f"\n✅ Enterprise Lens V4 — Models Ready\n"
305
- f" Device : {self.device.upper()}\n"
306
- f" Face vectors : {FUSED_FACE_DIM}-D ({adaface_status})\n"
307
- f" Object vectors: 1536-D (SigLIP+DINOv2)\n"
308
- f" Quality gate : det_score ≥ {FACE_QUALITY_GATE}, face_px ≥ {MIN_FACE_SIZE}\n"
309
- )
310
-
311
- # ── AdaFace loader ───────────────────────────────────────────────
312
- def _load_adaface(self):
313
- """
314
- Load AdaFace IR-50 MS1MV2 from HuggingFace.
315
- Controlled by ENABLE_ADAFACE env var (default off).
316
-
317
- When disabled: ArcFace(512) + zeros(512) → 1024-D output.
318
- Zero-padding is cosine-neutral — the ArcFace half still carries
319
- full identity signal; padded zeros don't pull any direction.
320
-
321
- When enabled: ArcFace(512) + AdaFace(512) → 1024-D.
322
- AdaFace is quality-adaptive: blurry/low-quality face crops receive
323
- downweighted embeddings, improving retrieval precision.
324
- """
325
- if not ENABLE_ADAFACE:
326
- print("⚠️ AdaFace disabled (ENABLE_ADAFACE != 1) — using zero-padded 1024-D")
327
- return
328
-
329
- import os, sys
330
- REPO_ID = "minchul/cvlface_adaface_ir50_ms1mv2"
331
- CACHE_PATH = os.path.expanduser("~/.cvlface_cache/minchul/cvlface_adaface_ir50_ms1mv2")
332
- try:
333
- from huggingface_hub import hf_hub_download
334
- from transformers import AutoModel as _HFAutoModel
335
-
336
- print("📦 Loading AdaFace IR-50 MS1MV2...")
337
- os.makedirs(CACHE_PATH, exist_ok=True)
338
-
339
- hf_hub_download(repo_id=REPO_ID, filename="files.txt",
340
- token=HF_TOKEN, local_dir=CACHE_PATH,
341
- local_dir_use_symlinks=False)
342
- with open(os.path.join(CACHE_PATH, "files.txt")) as f:
343
- extra = [x.strip() for x in f.read().split("\n") if x.strip()]
344
- for fname in extra + ["config.json", "wrapper.py", "model.safetensors"]:
345
- fpath = os.path.join(CACHE_PATH, fname)
346
- if not os.path.exists(fpath):
347
- hf_hub_download(repo_id=REPO_ID, filename=fname,
348
- token=HF_TOKEN, local_dir=CACHE_PATH,
349
- local_dir_use_symlinks=False)
350
-
351
- cwd = os.getcwd()
352
- os.chdir(CACHE_PATH)
353
- sys.path.insert(0, CACHE_PATH)
354
- try:
355
- model = _HFAutoModel.from_pretrained(
356
- CACHE_PATH, trust_remote_code=True, token=HF_TOKEN)
357
- finally:
358
- os.chdir(cwd)
359
- if CACHE_PATH in sys.path:
360
- sys.path.remove(CACHE_PATH)
361
-
362
- model = model.to(self.device).eval()
363
- with torch.no_grad():
364
- out = model(torch.zeros(1, 3, 112, 112).to(self.device))
365
- emb = out if isinstance(out, torch.Tensor) else out.embedding
366
- assert emb.shape[-1] == ADAFACE_DIM, f"Expected {ADAFACE_DIM}-D, got {emb.shape[-1]}"
367
-
368
- self.adaface_model = model
369
- print("✅ AdaFace IR-50 loaded — 1024-D FULL FUSION active")
370
-
371
- except Exception as e:
372
- print(f"⚠️ AdaFace load failed: {e} — falling back to zero-padded 1024-D")
373
- self.adaface_model = None
374
-
375
- # ── AdaFace inference ────────────────────────────────────────────
376
- def _adaface_embed(self, face_arr_chw: np.ndarray | None) -> np.ndarray | None:
377
- """
378
- Run AdaFace on a preprocessed (3, 112, 112) float32 CHW array.
379
- Returns a 512-D L2-normalised numpy embedding, or None on failure.
380
-
381
- The cvlface model may return a raw tensor or an object with .embedding —
382
- both output formats are handled here.
383
- """
384
- if self.adaface_model is None or face_arr_chw is None:
385
- return None
386
- try:
387
- t = torch.from_numpy(face_arr_chw).unsqueeze(0).to(self.device)
388
- if self.device == "cuda":
389
- t = t.half()
390
- with torch.no_grad():
391
- out = self.adaface_model(t)
392
- emb = out if isinstance(out, torch.Tensor) else out.embedding
393
- return F.normalize(emb.float(), p=2, dim=1)[0].cpu().numpy()
394
- except Exception as e:
395
- print(f"⚠️ AdaFace inference error: {e}")
396
- return None
397
-
398
- # ── Object lane: batched embedding ──────────────────────────────
399
- def _embed_crops_batch(self, crops: list[Image.Image]) -> list[np.ndarray]:
400
- """
401
- Embed a batch of PIL images through SigLIP and DINOv2, fuse results.
402
-
403
- SigLIP captures semantic/language-aligned meaning ("a red sports car").
404
- DINOv2 captures fine-grained visual texture and structure (self-supervised).
405
- Fusing both gives vectors that are sensitive to BOTH what something IS
406
- and what it LOOKS LIKE — better retrieval than either model alone.
407
-
408
- Why batch? GPUs process many inputs in parallel almost as fast as one.
409
- Why torch.no_grad()? Skips gradient graph construction — ~30 % faster,
410
- significant memory saving at inference time.
411
- Why F.normalize (L2)? Projects embeddings onto unit sphere.
412
- On the unit sphere: cosine_similarity = dot_product
413
- (cheaper and numerically stable).
414
- Also ensures neither SigLIP nor DINOv2 dominates
415
- the fused vector due to scale differences.
416
- """
417
- if not crops:
418
- return []
419
- with torch.no_grad():
420
- # SigLIP
421
- sig_in = self.siglip_processor(images=crops, return_tensors="pt", padding=True)
422
- sig_in = {k: v.to(self.device) for k, v in sig_in.items()}
423
- if self.device == "cuda":
424
- sig_in = {k: v.half() if v.dtype == torch.float32 else v
425
- for k, v in sig_in.items()}
426
- sig_out = self.siglip_model.get_image_features(**sig_in)
427
- # Handle all output types across transformers versions
428
- if hasattr(sig_out, "image_embeds"):
429
- sig_out = sig_out.image_embeds
430
- elif hasattr(sig_out, "pooler_output"):
431
- sig_out = sig_out.pooler_output
432
- elif hasattr(sig_out, "last_hidden_state"):
433
- sig_out = sig_out.last_hidden_state[:, 0, :]
434
- elif isinstance(sig_out, tuple):
435
- sig_out = sig_out[0]
436
- sig_vecs = F.normalize(sig_out.float(), p=2, dim=1).cpu()
437
-
438
- # DINOv2 — [:, 0, :] extracts the [CLS] token which aggregates
439
- # the global image representation across the entire sequence
440
- dino_in = self.dinov2_processor(images=crops, return_tensors="pt")
441
- dino_in = {k: v.to(self.device) for k, v in dino_in.items()}
442
- if self.device == "cuda":
443
- dino_in = {k: v.half() if v.dtype == torch.float32 else v
444
- for k, v in dino_in.items()}
445
- dino_out = self.dinov2_model(**dino_in)
446
- dino_vecs = F.normalize(
447
- dino_out.last_hidden_state[:, 0, :].float(), p=2, dim=1).cpu()
448
-
449
- fused = F.normalize(torch.cat([sig_vecs, dino_vecs], dim=1), p=2, dim=1)
450
-
451
- return [fused[i].numpy() for i in range(len(crops))]
452
-
453
- # ── Face lane: detection + dual encoding ─────────────────────────
454
- def _detect_and_encode_faces(self, img_np: np.ndarray) -> list[dict]:
455
- """
456
- Detect all faces using InsightFace SCRFD-10GF at multiple scales,
457
- encode each face with ArcFace-R100 + AdaFace IR-50, and return
458
- 1024-D fused vectors.
459
-
460
- Pipeline per face:
461
- 1. ArcFace-R100 (512-D) from InsightFace's built-in recognition
462
- 2. AdaFace IR-50 (512-D) from separately loaded model
463
- 3. Concatenate + L2-normalise → 1024-D final vector
464
- 4. Quality gates: det_score ≥ FACE_QUALITY_GATE, width ≥ MIN_FACE_SIZE
465
-
466
- Multi-scale strategy:
467
- - Run SCRFD at 1280, 960, and 640 px.
468
- - Run once more on horizontally flipped image (catches turned faces).
469
- - Merge all detections and deduplicate by IoU.
470
- Rationale: a face that's 15 px at 640 becomes 30 px at 1280;
471
- the detector finds it at the larger scale.
472
-
473
- AdaFace unavailable:
474
- Zero-pad to maintain 1024-D. The ArcFace half carries full identity
475
- signal; zero padding is cosine-neutral (no direction bias).
476
-
477
- Returns list of dicts:
478
- { type, vector (1024-D), face_idx, bbox, face_crop, det_score, face_width_px }
479
- """
480
- if self.face_app is None:
481
- return []
482
-
483
- try:
484
- if img_np.dtype != np.uint8:
485
- img_np = (img_np * 255).astype(np.uint8)
486
- bgr = img_np[:, :, ::-1].copy() if img_np.shape[2] == 3 else img_np.copy()
487
-
488
- # CLAHE: boost contrast on dark/backlit/low-contrast photos
489
- bgr_enhanced = _clahe_enhance(bgr)
490
-
491
- # Multi-scale detection — bboxes are scaled back to original coords
492
- all_raw_faces = []
493
- H, W = bgr.shape[:2]
494
-
495
- for scale in DET_SCALES:
496
- scale_w = min(W, scale[0])
497
- scale_h = min(H, scale[1])
498
- bgr_scaled = (
499
- bgr_enhanced if scale_w == W and scale_h == H
500
- else cv2.resize(bgr_enhanced, (scale_w, scale_h))
501
- )
502
- try:
503
- self.face_app.det_model.input_size = scale
504
- with self._face_lock:
505
- faces_at_scale = self.face_app.get(bgr_scaled)
506
- sx, sy = W / scale_w, H / scale_h
507
- for f in faces_at_scale:
508
- if sx != 1.0 or sy != 1.0:
509
- f.bbox[0] *= sx; f.bbox[1] *= sy
510
- f.bbox[2] *= sx; f.bbox[3] *= sy
511
- all_raw_faces.extend(faces_at_scale)
512
- except Exception:
513
- pass
514
-
515
- # Horizontal-flip pass — catches profile/turned faces
516
- bgr_flip = cv2.flip(bgr_enhanced, 1)
517
- try:
518
- self.face_app.det_model.input_size = DET_SIZE_PRIMARY
519
- with self._face_lock:
520
- faces_flip = self.face_app.get(bgr_flip)
521
- for f in faces_flip:
522
- x1, y1, x2, y2 = f.bbox
523
- f.bbox[0] = W - x2
524
- f.bbox[2] = W - x1
525
- all_raw_faces.extend(faces_flip)
526
- except Exception:
527
- pass
528
-
529
- # Restore primary size
530
- self.face_app.det_model.input_size = DET_SIZE_PRIMARY
531
-
532
- faces = _dedup_faces(all_raw_faces)
533
- print(f" Raw detections: {len(all_raw_faces)} → after dedup: {len(faces)}")
534
-
535
- results = []
536
- accepted = 0
537
-
538
- for idx, face in enumerate(faces):
539
- if accepted >= MAX_FACES_PER_IMAGE:
540
- break
541
-
542
- bbox_raw = face.bbox.astype(int)
543
- x1, y1, x2, y2 = bbox_raw
544
- x1 = max(0, x1); y1 = max(0, y1)
545
- x2 = min(bgr.shape[1], x2); y2 = min(bgr.shape[0], y2)
546
- w, h = x2 - x1, y2 - y1
547
- if w <= 0 or h <= 0:
548
- continue
549
-
550
- # Quality gate 1: minimum pixel size
551
- if w < MIN_FACE_SIZE or h < MIN_FACE_SIZE:
552
- print(f" Face {idx}: SKIP — too small ({w}×{h}px)")
553
- continue
554
-
555
- # Quality gate 2: detector confidence
556
- det_score = float(face.det_score) if hasattr(face, "det_score") else 1.0
557
- if det_score < FACE_QUALITY_GATE:
558
- print(f" Face {idx}: SKIP — low det_score ({det_score:.3f})")
559
- continue
560
-
561
- if face.embedding is None:
562
- continue
563
-
564
- # ArcFace embedding (built into InsightFace buffalo_l)
565
- arcface_vec = face.embedding.astype(np.float32)
566
- n = np.linalg.norm(arcface_vec)
567
- if n > 0:
568
- arcface_vec = arcface_vec / n
569
-
570
- # AdaFace embedding (quality-adaptive)
571
- face_chw = _face_crop_for_adaface(bgr, x1, y1, x2, y2)
572
- adaface_vec = self._adaface_embed(face_chw)
573
-
574
- # Fuse to 1024-D — always output FUSED_FACE_DIM regardless of AdaFace status
575
- if adaface_vec is not None:
576
- fused_raw = np.concatenate([arcface_vec, adaface_vec])
577
- else:
578
- fused_raw = np.concatenate([arcface_vec,
579
- np.zeros(ADAFACE_DIM, dtype=np.float32)])
580
- n2 = np.linalg.norm(fused_raw)
581
- final_vec = (fused_raw / n2) if n2 > 0 else fused_raw
582
-
583
- face_crop_b64 = _crop_to_b64(bgr, x1, y1, x2, y2)
584
-
585
- results.append({
586
- "type": "face",
587
- "vector": final_vec,
588
- "face_idx": accepted,
589
- # bbox exposed so the frontend can draw boxes on the query image
590
- "bbox": [int(x1), int(y1), int(w), int(h)],
591
- "face_crop": face_crop_b64,
592
- "det_score": det_score,
593
- "face_width_px": int(w),
594
- })
595
- accepted += 1
596
- print(f" Face {idx}: ✅ ACCEPTED — {w}×{h}px | det={det_score:.3f}")
597
-
598
- print(f"👤 {accepted} face(s) passed quality gate")
599
- return results
600
-
601
- except Exception as e:
602
- print(f"🟠 InsightFace error: {e}\n{traceback.format_exc()[-600:]}")
603
- return []
604
-
605
- # ── Main pipeline ────────────────────────────────────────────────
606
- def process_image(
607
- self,
608
- image_path: str,
609
- detect_faces: bool = True,
610
- ) -> list[dict]:
611
- """
612
- Full inference pipeline for a single image.
613
-
614
- Always runs both lanes:
615
- Face → list of { type:"face", vector(1024-D), face_idx, bbox,
616
- face_crop, det_score, face_width_px }
617
- Object → list of { type:"object", vector(1536-D) }
618
-
619
- main.py decides which lane's results to use for Pinecone operations
620
- based on the endpoint context (upload stores both; search can use both).
621
-
622
- Cache strategy:
623
- Key = (md5_of_first_64KB, detect_faces)
624
- Hit → return cached result immediately (skips all model inference)
625
- Miss → run pipeline, cache result, evict LRU entry if over capacity
626
-
627
- Cache is protected by _cache_lock (threading.Lock) to prevent race
628
- conditions when MAX_CONCURRENT_INFERENCES > 1.
629
- """
630
- cache_key = f"{img_hash(image_path)}_{detect_faces}"
631
-
632
- with self._cache_lock:
633
- if cache_key in self._cache:
634
- print("⚡ Cache hit")
635
- return self._cache[cache_key]
636
-
637
- extracted = []
638
- original_pil = Image.open(image_path).convert("RGB")
639
- img_np = np.array(original_pil) # RGB uint8, full resolution
640
- faces_found = False
641
-
642
- # ── Face lane ────────────────────────────────────────────
643
- if detect_faces and self.face_app is not None:
644
- face_results = self._detect_and_encode_faces(img_np)
645
- if face_results:
646
- faces_found = True
647
- extracted.extend(face_results)
648
-
649
- # ── Object lane ──────────────────────────────────────────
650
- # Always runs, even when faces are found.
651
- # Person-class YOLO crops are skipped when face lane is active
652
- # to avoid embedding the same person twice.
653
- #
654
- # Crop 0 is always the full (resized) image — ensures we always
655
- # have at least one embedding even if YOLO finds nothing.
656
- # YOLO is given the already-loaded PIL image to avoid re-reading
657
- # the file from disk.
658
- crops: list[Image.Image] = []
659
- yolo_results = self.yolo(original_pil, conf=YOLO_CONF_THRESHOLD, verbose=False)
660
-
661
- for r in yolo_results:
662
- if r.masks is not None:
663
- for seg_idx, mask_xy in enumerate(r.masks.xy):
664
- cls_id = int(r.boxes.cls[seg_idx].item())
665
- if faces_found and cls_id == YOLO_PERSON_CLASS_ID:
666
- continue
667
- polygon = np.array(mask_xy, dtype=np.int32)
668
- if len(polygon) < 3:
669
- continue
670
- x, y, w, h = cv2.boundingRect(polygon)
671
- if w < YOLO_MIN_CROP_PX or h < YOLO_MIN_CROP_PX:
672
- continue
673
- crops.append(original_pil.crop((x, y, x + w, y + h)))
674
- if len(crops) >= MAX_CROPS:
675
- break
676
- elif r.boxes is not None:
677
- for box in r.boxes:
678
- cls_id = int(box.cls.item())
679
- if faces_found and cls_id == YOLO_PERSON_CLASS_ID:
680
- continue
681
- x1, y1, x2, y2 = box.xyxy[0].tolist()
682
- if (x2 - x1) < YOLO_MIN_CROP_PX or (y2 - y1) < YOLO_MIN_CROP_PX:
683
- continue
684
- crops.append(original_pil.crop((x1, y1, x2, y2)))
685
- if len(crops) >= MAX_CROPS:
686
- break
687
-
688
- # Prepend the full image as crop 0, then resize ALL crops uniformly.
689
- # (Previously the full image was pre-resized before appending, causing
690
- # _resize_pil to be called on it twice. Now we resize everything once.)
691
- all_crops = [original_pil] + crops
692
- all_crops = [_resize_pil(c, MAX_IMAGE_SIZE) for c in all_crops]
693
-
694
- print(f"🧠 Embedding {len(all_crops)} object crop(s)...")
695
- obj_vecs = self._embed_crops_batch(all_crops)
696
- extracted.extend({"type": "object", "vector": v} for v in obj_vecs)
697
-
698
- # Cache with lock — prevents concurrent writes from corrupting eviction
699
- with self._cache_lock:
700
- if len(self._cache) >= INFERENCE_CACHE_SIZE:
701
- # Evict LRU entry (first inserted key in plain dict = oldest)
702
- oldest = next(iter(self._cache))
703
- del self._cache[oldest]
704
- self._cache[cache_key] = extracted
705
-
706
- return extracted
707
-
708
- async def process_image_async(
709
- self,
710
- image_path: str,
711
- detect_faces: bool = True,
712
- ) -> list[dict]:
713
- """
714
- Async wrapper for process_image — offloads blocking inference to a
715
- thread-pool executor so FastAPI's event loop remains responsive.
716
-
717
- functools.partial is used instead of a lambda to make the call
718
- picklable, which some executor backends require.
719
- """
720
- loop = asyncio.get_event_loop()
721
- return await loop.run_in_executor(
722
- None,
723
- functools.partial(self.process_image, image_path, detect_faces),
724
- )