feat: add frontend system health diagnostics and mute HF Celery gossip

backend/clean_db.py

@@ -17,7 +17,10 @@ async def main():
     try:
         q = QdrantClient(url=settings.qdrant_url, api_key=settings.qdrant_api_key)
         q.delete_collection(settings.collection_name)
-        q.create_collection(settings.collection_name, vectors_config=VectorParams(size=384, distance=Distance.COSINE))
+        q.create_collection(
+            collection_name=settings.collection_name, 
+            vectors_config=VectorParams(size=settings.vector_size, distance=Distance.COSINE)
+        )
         
         # Reinject indices required natively by the pipeline
         q.create_payload_index(

backend/src/workers/celery_app.py

@@ -4,6 +4,14 @@ from ..config import get_settings
 
 settings = get_settings()
 
+# Connection pool options
+_REDIS_TRANSPORT_OPTS = {
+    "max_connections": 10,
+    "socket_keepalive": True,
+    "socket_connect_timeout": 10,
+    "retry_on_timeout": True,
+}
+
 celery_app = Celery(
     "talentpulse",
     broker=settings.redis_url,
@@ -17,8 +25,17 @@ celery_app.conf.update(
     result_serializer="json",
     timezone="UTC",
     enable_utc=True,
-    task_track_started=True,
-    result_expires=3600,
-    worker_prefetch_multiplier=1,
-    task_acks_late=True,
+    
+    # ----------------------------------------------------
+    # FREE TIER REDIS OPTIMIZATIONS (Max 30 Connections)
+    # ----------------------------------------------------
+    worker_send_task_events=False,        # Disable task events (saves connections)
+    worker_enable_remote_control=False,   # Disable mingle/broadcast (saves 2 conns per worker)
+    task_track_started=False,             # Disable state tracking noise
+    result_expires=1800,
+    
+    # Limit broker & backend pool
+    broker_transport_options=_REDIS_TRANSPORT_OPTS,
+    redis_max_connections=10,
+    broker_pool_limit=10,
 )

backend/src/workers/ingest.py

@@ -99,6 +99,18 @@ async def _ingest_candidates_async(rows: list[dict], session_id: str | None) ->
         texts.append(candidate_text)
 
     embeddings = embed_texts(texts)
+    
+    # Validation: Ensure generated embeddings match the configured vector_size for Qdrant
+    actual_dim = embeddings.shape[1]
+    expected_dim = settings.vector_size
+    print(f"[INGEST] Generated {len(texts)} embeddings with dimension {actual_dim} (Expected: {expected_dim})")
+    
+    if actual_dim != expected_dim:
+        raise ValueError(
+            f"Vector size mismatch! Model '{settings.embedding_model}' produced dimension {actual_dim}, "
+            f"but Qdrant collection '{settings.collection_name}' expects {expected_dim}. "
+            f"Please check your EMBEDDING_MODEL and VECTOR_SIZE settings."
+        )
 
     async with AsyncSessionLocal() as session:
         qdrant_points = []