added redis hammer rate limiting

app/service/vector_service.py

@@ -220,39 +220,45 @@ class VectorService:
         metadata: List[Dict[str, Any]],
         namespace: str
     ):
-        """Store in Redis with 24h TTL (non-blocking)"""
+        """
+        🛡️ **Redis storage - NON-BLOCKING with rate limiting**
+        Processes in batches with small delays to prevent overwhelming Redis
+        """
         try:
             stored = 0
-        
-            # Process in smaller batches to reduce HTTP calls
-            batch_size = 50
+            batch_size = 50  # Store 50 at a time
         
             for i in range(0, len(embeddings), batch_size):
                 batch_embeddings = embeddings[i:i + batch_size]
                 batch_metadata = metadata[i:i + batch_size]
             
-                # Store batch sequentially but non-blocking
+                # Process batch with concurrent tasks (max 10 at once)
+                tasks = []
                 for idx, (emb, meta) in enumerate(zip(batch_embeddings, batch_metadata)):
                     global_idx = i + idx
                     key = f"vector:{namespace}:{global_idx}:{int(time.time())}"
                 
-                    # ✅ FIX: Run in thread pool to avoid blocking
-                    await asyncio.to_thread(
+                    # Create task for non-blocking Redis call
+                    task = asyncio.to_thread(
                         event_hub.setex,
                         key,
-                        86400,
+                        86400,  # 24 hours
                         json.dumps({
                             "embedding": emb,
                             "metadata": meta,
                             "org_id": self.org_id
                         })
                     )
-                    stored += 1
+                    tasks.append(task)
             
-                # Small delay every batch to prevent overwhelming Redis
+                # Run batch concurrently
+                await asyncio.gather(*tasks, return_exceptions=True)
+                stored += len(batch_embeddings)
+            
+                # ✅ **RATE LIMITING**: Sleep every 200 vectors
                 if i > 0 and i % 200 == 0:
-                    await asyncio.sleep(0.01)
-        
+                    await asyncio.sleep(0.01)  # 10ms pause
+            
             logger.info(f"[✅ VECTOR] Redis: Stored {stored} vectors (non-blocking)")
         
         except Exception as e:

app/tasks/analytics_worker.py

@@ -486,7 +486,7 @@ class AnalyticsWorker:
             
             # 3️⃣ Store in vector service (Redis + DuckDB VSS)
             namespace = f"{self._entity_type}:{self.org_id}"
-            vector_service.upsert_embeddings(
+            await vector_service.upsert_embeddings(
                 embeddings=embeddings,
                 metadata=metadata,
                 namespace=namespace