Spaces:
Runtime error
Runtime error
File size: 6,103 Bytes
67ac4d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | """
AttendX Face Recognition Pipeline Calibration & Evaluation Tool
Evaluates Genuine (same student) and Impostor (different student) similarity distributions,
calculates FAR/FRR metrics across thresholds, and outputs a detailed calibration report.
"""
import os
import sys
import json
import numpy as np
# Add backend directory to sys.path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from app.database.session import SessionLocal
from app.models.models import Student, FaceEmbedding
from app.core.config import settings
def run_evaluation():
db = SessionLocal()
students = db.query(Student).all()
# Collect embeddings per student
student_embs = {}
for s in students:
embs = [np.array(e.embedding, dtype=np.float32) for e in s.embeddings if e.embedding]
# Normalize
normalized = []
for vec in embs:
norm = np.linalg.norm(vec)
if norm > 0:
normalized.append(vec / norm)
if normalized:
student_embs[s.id] = {
"name": s.name,
"student_id": s.student_id,
"embeddings": normalized
}
print(f"\n=======================================================")
print(f" AttendX Face Recognition Calibration Benchmark")
print(f" Loaded {len(student_embs)} students with registered embeddings")
print(f"=======================================================\n")
genuine_scores = []
impostor_scores = []
student_ids = list(student_embs.keys())
# 1. Genuine comparisons (same student, pair-wise)
for s_id in student_ids:
embs = student_embs[s_id]["embeddings"]
for i in range(len(embs)):
for j in range(i + 1, len(embs)):
sim = float(np.dot(embs[i], embs[j]))
genuine_scores.append(sim)
# 2. Impostor comparisons (cross-student pairs)
for i in range(len(student_ids)):
for j in range(i + 1, len(student_ids)):
s1_embs = student_embs[student_ids[i]]["embeddings"]
s2_embs = student_embs[student_ids[j]]["embeddings"]
for e1 in s1_embs:
for e2 in s2_embs:
sim = float(np.dot(e1, e2))
impostor_scores.append(sim)
if not genuine_scores:
print("[Note] Not enough multi-image genuine pairs in current DB to compute intra-class variance.")
# Self-identity fallback baseline
genuine_scores = [1.0]
gen_arr = np.array(genuine_scores)
imp_arr = np.array(impostor_scores) if impostor_scores else np.array([0.0])
gen_stats = {
"count": len(genuine_scores),
"min": float(np.min(gen_arr)),
"max": float(np.max(gen_arr)),
"mean": float(np.mean(gen_arr)),
"median": float(np.median(gen_arr)),
"std": float(np.std(gen_arr))
}
imp_stats = {
"count": len(impostor_scores),
"min": float(np.min(imp_arr)),
"max": float(np.max(imp_arr)),
"mean": float(np.mean(imp_arr)),
"median": float(np.median(imp_arr)),
"std": float(np.std(imp_arr))
}
print("GENUINE MATCHES (Same Student):")
print(f" Count: {gen_stats['count']}")
print(f" Min: {gen_stats['min']:.4f}")
print(f" Max: {gen_stats['max']:.4f}")
print(f" Mean: {gen_stats['mean']:.4f}")
print(f" Median: {gen_stats['median']:.4f}")
print(f" StdDev: {gen_stats['std']:.4f}\n")
print("IMPOSTOR MATCHES (Different Students):")
print(f" Count: {imp_stats['count']}")
print(f" Min: {imp_stats['min']:.4f}")
print(f" Max: {imp_stats['max']:.4f}")
print(f" Mean: {imp_stats['mean']:.4f}")
print(f" Median: {imp_stats['median']:.4f}")
print(f" StdDev: {imp_stats['std']:.4f}\n")
# Evaluate across various candidate thresholds
threshold_grid = [0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60]
sweep_results = []
print(f"{'Threshold':<10} | {'FAR (%)':<10} | {'FRR (%)':<10} | {'Precision (%)':<15} | {'Recall (%)':<12}")
print("-" * 65)
for thresh in threshold_grid:
# False Accept: Impostor >= thresh
fa_count = np.sum(imp_arr >= thresh)
far = (fa_count / len(imp_arr)) * 100 if len(imp_arr) > 0 else 0.0
# False Reject: Genuine < thresh
fr_count = np.sum(gen_arr < thresh)
frr = (fr_count / len(gen_arr)) * 100 if len(gen_arr) > 0 else 0.0
# True Positive & False Positive
tp = np.sum(gen_arr >= thresh)
fp = fa_count
precision = (tp / (tp + fp)) * 100 if (tp + fp) > 0 else 100.0
recall = (tp / len(gen_arr)) * 100 if len(gen_arr) > 0 else 0.0
sweep_results.append({
"threshold": thresh,
"far_percent": round(float(far), 2),
"frr_percent": round(float(frr), 2),
"precision_percent": round(float(precision), 2),
"recall_percent": round(float(recall), 2)
})
is_current = " (CURRENT MATCH)" if thresh == settings.FACE_MATCH_THRESHOLD else (" (CURRENT REVIEW)" if thresh == settings.FACE_REVIEW_THRESHOLD else "")
print(f"{thresh:<10.2f} | {far:<10.2f} | {frr:<10.2f} | {precision:<15.2f} | {recall:<12.2f}{is_current}")
# Export report to JSON
report = {
"genuine_statistics": gen_stats,
"impostor_statistics": imp_stats,
"threshold_sweep": sweep_results,
"active_configuration": {
"FACE_MATCH_THRESHOLD": settings.FACE_MATCH_THRESHOLD,
"FACE_REVIEW_THRESHOLD": settings.FACE_REVIEW_THRESHOLD,
"FACE_MIN_MARGIN": settings.FACE_MIN_MARGIN,
"FACE_BLUR_THRESHOLD": settings.FACE_BLUR_THRESHOLD,
"FACE_MIN_SIZE": settings.FACE_MIN_SIZE
}
}
report_path = os.path.join(settings.EXPORTS_DIR, "face_evaluation_report.json")
with open(report_path, "w") as f:
json.dump(report, f, indent=2)
print(f"\nReport successfully exported to: {report_path}\n")
if __name__ == "__main__":
run_evaluation()
|