Spaces:
Sleeping
Sleeping
| from fastapi import APIRouter, UploadFile, File, HTTPException, BackgroundTasks, Form | |
| from fastapi.concurrency import run_in_threadpool | |
| from app.services.face_service import face_service | |
| from app.services.object_service import detector as object_service | |
| from app.services.memory_service import memory_service | |
| import shutil | |
| from pathlib import Path | |
| import uuid | |
| from typing import Dict, Any | |
| import base64 | |
| from PIL import Image | |
| import io | |
| router = APIRouter() | |
| TEMP_DIR = Path("temp_uploads") | |
| TEMP_DIR.mkdir(exist_ok=True) | |
| # Ensure enrollment dir exists | |
| ENROLL_DIR = Path("photo/enrolled") | |
| ENROLL_DIR.mkdir(parents=True, exist_ok=True) | |
| def encode_image_base64(image_path: str): | |
| """Resize and encode image to base64 for storage.""" | |
| try: | |
| with Image.open(image_path) as img: | |
| # Resize to thumbnail to save space (e.g., 300px max) | |
| img.thumbnail((300, 300)) | |
| buffered = io.BytesIO() | |
| img.convert("RGB").save(buffered, format="JPEG", quality=70) | |
| img_str = base64.b64encode(buffered.getvalue()).decode("utf-8") | |
| return f"data:image/jpeg;base64,{img_str}" | |
| except Exception as e: | |
| print(f"Error encoding image: {e}") | |
| return None | |
| async def recognize_person(background_tasks: BackgroundTasks, file: UploadFile = File(...)): | |
| """ | |
| Receive an image, detect faces, search Qdrant for identity. | |
| """ | |
| # 1. Save temp file | |
| file_id = str(uuid.uuid4()) | |
| ext = Path(file.filename).suffix | |
| temp_path = TEMP_DIR / f"{file_id}{ext}" | |
| try: | |
| with open(temp_path, "wb") as buffer: | |
| shutil.copyfileobj(file.file, buffer) | |
| # 2. Generate Embedding | |
| # Note: FaceService currently returns list of floats or empty list | |
| embedding = await run_in_threadpool(face_service.generate_embedding, str(temp_path)) | |
| if not embedding: | |
| return {"status": "no_face_detected", "person": None} | |
| # 3. Search Memory | |
| matches = await run_in_threadpool(memory_service.search_face, embedding) | |
| if matches: | |
| best_match = matches[0] | |
| # Check threshold (Cosine Similarity > 0.4 implies match) | |
| if best_match.score > 0.4: | |
| name = best_match.payload.get("name", "Unknown") | |
| relation = best_match.payload.get("relation", "Unknown") | |
| notes = best_match.payload.get("notes", "") | |
| # TTS Feedback | |
| greeting = f"Hello {name}." | |
| if notes: | |
| greeting += f" {notes}" | |
| elif relation != "Unknown": | |
| greeting += f" You are a {relation}." | |
| return { | |
| "status": "identified", | |
| "person": { | |
| "name": name, | |
| "relation": relation, | |
| "confidence": best_match.score, | |
| "id": best_match.payload.get("person_id"), | |
| "notes": notes, | |
| "image": best_match.payload.get("image_base64", None), | |
| "audio": best_match.payload.get("audio_base64", None) | |
| } | |
| } | |
| return {"status": "unknown", "person": None} | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| finally: | |
| # Cleanup | |
| if temp_path.exists(): | |
| temp_path.unlink() | |
| async def remember_person( | |
| background_tasks: BackgroundTasks, | |
| name: str = Form(...), | |
| relation: str = Form("Acquaintance"), | |
| notes: str = Form(None), | |
| age: int = Form(None), | |
| file: UploadFile = File(...), | |
| audio_file: UploadFile = File(None) | |
| ): | |
| """Enroll a new person with optional voice sample.""" | |
| file_id = str(uuid.uuid4()) | |
| filename = f"{name.replace(' ', '_')}_{file_id}.jpg" | |
| perm_path = ENROLL_DIR / filename | |
| # Audio Path | |
| audio_b64 = None | |
| if audio_file: | |
| audio_path = Path("audio/enrolled") / f"{name.replace(' ', '_')}_{file_id}.webm" | |
| audio_path.parent.mkdir(parents=True, exist_ok=True) | |
| try: | |
| with open(audio_path, "wb") as buffer: | |
| shutil.copyfileobj(audio_file.file, buffer) | |
| # Encode for Cloud | |
| with open(audio_path, "rb") as f: | |
| audio_b64 = base64.b64encode(f.read()).decode("utf-8") | |
| except Exception as e: | |
| print(f"Error saving audio: {e}") | |
| try: | |
| # Save Image locally as backup | |
| with open(perm_path, "wb") as buffer: | |
| shutil.copyfileobj(file.file, buffer) | |
| # Generate Embedding | |
| embedding = await run_in_threadpool(face_service.generate_embedding, str(perm_path)) | |
| if not embedding: | |
| perm_path.unlink() | |
| raise HTTPException(status_code=400, detail="No face detected in enrollment photo. Please look at the camera.") | |
| # Encode Image for Cloud Storage | |
| img_b64 = encode_image_base64(str(perm_path)) | |
| # 4. Generate Avatar | |
| from app.services.avatar_service import avatar_service | |
| avatar_url = avatar_service.generate_avatar(str(perm_path)) | |
| # Store in Qdrant | |
| metadata = { | |
| "name": name, | |
| "relation": relation, | |
| "age": age, | |
| "type": "person", | |
| "notes": notes or f"This is {name}, your {relation}.", | |
| "image_base64": img_b64, | |
| "avatar_url": avatar_url | |
| } | |
| if audio_b64: | |
| metadata["audio_base64"] = audio_b64 # Store voice sample in cloud! | |
| await run_in_threadpool(memory_service.store_face_memory, | |
| person_id=name.replace(" ", "_"), | |
| embedding=embedding, | |
| metadata=metadata | |
| ) | |
| msg = f"I have enrolled {name}." | |
| return {"status": "stored", "name": name, "avatar_url": avatar_url} | |
| except Exception as e: | |
| if perm_path.exists(): | |
| perm_path.unlink() | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| async def remember_patient( | |
| background_tasks: BackgroundTasks, | |
| name: str = Form(...), | |
| relation: str = Form("Acquaintance"), | |
| notes: str = Form(None), | |
| age: int = Form(None), | |
| file: UploadFile = File(...), | |
| audio_file: UploadFile = File(None) | |
| ): | |
| """Enroll a new PATIENT/Person via Caregiver (Stored in 'patients' collection)""" | |
| file_id = str(uuid.uuid4()) | |
| filename = f"{name.replace(' ', '_')}_{file_id}.jpg" | |
| perm_path = ENROLL_DIR / filename | |
| # Audio Path | |
| audio_b64 = None | |
| if audio_file: | |
| audio_path = Path("audio/enrolled") / f"{name.replace(' ', '_')}_{file_id}.webm" | |
| audio_path.parent.mkdir(parents=True, exist_ok=True) | |
| try: | |
| with open(audio_path, "wb") as buffer: | |
| shutil.copyfileobj(audio_file.file, buffer) | |
| with open(audio_path, "rb") as f: | |
| audio_b64 = base64.b64encode(f.read()).decode("utf-8") | |
| except Exception as e: | |
| print(f"Error saving audio: {e}") | |
| try: | |
| # Save Image | |
| with open(perm_path, "wb") as buffer: | |
| shutil.copyfileobj(file.file, buffer) | |
| # Generate Embedding | |
| embedding = await run_in_threadpool(face_service.generate_embedding, str(perm_path)) | |
| if not embedding: | |
| perm_path.unlink() | |
| return {"status": "error", "message": "No face detected in enrollment photo."} | |
| # Encode for storage | |
| img_b64 = encode_image_base64(str(perm_path)) | |
| # Generate Avatar | |
| from app.services.avatar_service import avatar_service | |
| avatar_url = avatar_service.generate_avatar(str(perm_path)) | |
| # Store in Qdrant PATIENTS collection | |
| metadata = { | |
| "name": name, | |
| "relation": relation, | |
| "age": age, | |
| "type": "patient_contact", | |
| "notes": notes or f"This is {name}, your {relation}.", | |
| "image_base64": img_b64, | |
| "avatar_url": avatar_url | |
| } | |
| if audio_b64: | |
| metadata["audio_base64"] = audio_b64 | |
| await run_in_threadpool(memory_service.store_patient_memory, | |
| person_id=name.replace(" ", "_"), | |
| embedding=embedding, | |
| metadata=metadata | |
| ) | |
| return {"status": "stored", "name": name, "avatar_url": avatar_url} | |
| except Exception as e: | |
| if perm_path.exists(): | |
| perm_path.unlink() | |
| raise HTTPException(status_code=500, detail=str(e)) | |
| async def remember_object( | |
| background_tasks: BackgroundTasks, | |
| name: str = Form(...), | |
| notes: str = Form(None), | |
| file: UploadFile = File(...) | |
| ): | |
| """Register a new personal object (e.g. Medicine Box)""" | |
| file_id = str(uuid.uuid4()) | |
| temp_path = TEMP_DIR / f"{file_id}_{file.filename}" | |
| try: | |
| with open(temp_path, "wb") as buffer: | |
| shutil.copyfileobj(file.file, buffer) | |
| # Generate Embedding | |
| embedding = await run_in_threadpool(object_service.generate_embedding, str(temp_path)) | |
| # Encode | |
| img_b64 = encode_image_base64(str(temp_path)) | |
| # Store | |
| await run_in_threadpool(memory_service.store_object_memory, | |
| object_id=str(uuid.uuid4()), | |
| embedding=embedding, | |
| metadata={ | |
| "name": name, | |
| "type": "object", | |
| "notes": notes or f"This is your {name}.", | |
| "image_base64": img_b64 | |
| } | |
| ) | |
| msg = f"I have remembered your {name}." | |
| return {"status": "stored", "name": name} | |
| finally: | |
| if temp_path.exists(): | |
| temp_path.unlink() | |
| async def find_object(background_tasks: BackgroundTasks, file: UploadFile = File(...)): | |
| """Identify a specific personal object.""" | |
| file_id = str(uuid.uuid4()) | |
| temp_path = TEMP_DIR / f"{file_id}_{file.filename}" | |
| try: | |
| with open(temp_path, "wb") as buffer: | |
| shutil.copyfileobj(file.file, buffer) | |
| # 1. Generate Embedding | |
| embedding = await run_in_threadpool(object_service.generate_embedding, str(temp_path)) | |
| # 2. Search | |
| matches = await run_in_threadpool(memory_service.search_object, embedding) | |
| found_name = "Unknown Object" | |
| found_notes = "" | |
| found_img = None | |
| if matches and matches[0].score > 0.6: # Threshold | |
| best = matches[0] | |
| found_name = best.payload.get("name", "Unknown") | |
| found_notes = best.payload.get("notes", "") | |
| found_img = best.payload.get("image_base64", None) | |
| # TTS | |
| msg = f"This looks like your {found_name}." | |
| if found_notes: | |
| msg += f" {found_notes}" | |
| return { | |
| "status": "identified", | |
| "object": { | |
| "name": found_name, | |
| "notes": found_notes, | |
| "confidence": best.score, | |
| "location": best.payload.get("location", "Unknown"), | |
| "image": found_img | |
| } | |
| } | |
| # Fallback: YOLO Detection -> Auto-Enroll | |
| detections = await run_in_threadpool(object_service.detect_objects, str(temp_path)) | |
| if detections: | |
| # Found "cell phone", "bottle", etc. | |
| # Pick the highest confidence object | |
| best_det = max(detections, key=lambda x: x['confidence']) | |
| label = best_det['object'] | |
| # Auto-Learn: Store this specific instance embedding | |
| object_id = str(uuid.uuid4()) | |
| # Use the already calculated embedding | |
| # Note: We should ideally crop the object, but full image embedding is OK for prototype | |
| # if the object is dominant. | |
| # Encode for storage | |
| img_b64 = encode_image_base64(str(temp_path)) | |
| # Determine location (Mock or Current Context) | |
| # Since we don't have GPS, we say "Last Seen Location" or date | |
| from datetime import datetime | |
| timestamp = datetime.now().strftime("%I:%M %p") | |
| location = f"Last seen at {timestamp}" | |
| await run_in_threadpool(memory_service.store_object_memory, | |
| object_id=object_id, | |
| embedding=embedding, | |
| metadata={ | |
| "name": label, | |
| "type": "object", | |
| "notes": "Auto-enrolled from observation.", | |
| "location": location, | |
| "image_base64": img_b64 | |
| } | |
| ) | |
| found_name = label | |
| found_notes = "I just learned this object." | |
| # Return as 'identified' so Frontend treats it as a known object | |
| return { | |
| "status": "identified", | |
| "object": { | |
| "name": found_name, | |
| "notes": found_notes, | |
| "confidence": best_det['confidence'], | |
| "location": location, | |
| "image": img_b64 | |
| } | |
| } | |
| return {"status": "unknown", "object": None} | |
| finally: | |
| if temp_path.exists(): | |
| temp_path.unlink() | |
| async def debug_names(): | |
| """List all names in Qdrant Faces""" | |
| try: | |
| res = memory_service.client.scroll( | |
| collection_name="faces", | |
| limit=100, | |
| with_payload=True | |
| ) | |
| points = res[0] | |
| names = [p.payload.get("name") for p in points] | |
| return {"count": len(names), "names": names} | |
| except Exception as e: | |
| return {"error": str(e)} | |