imported time in mapper

app/mapper.py

@@ -8,6 +8,8 @@ from app.db import get_conn, ensure_raw_table
 from app.utils.detect_industry import _ALIAS, detect_industry
 # app/mapper.py (add line 1)
 from app.hybrid_entity_detector import hybrid_detect_entity_type
+import time
+
 
 # ----------------------  Canonical schema base  ---------------------- #
 CANONICAL = {

app/routers/datasources.py

@@ -108,7 +108,7 @@ async def create_source_json(
         # 1. 💾 Store raw data for audit & lineage
         bootstrap(orgId, payload.data)
         print(f"[api/json] ✅ Raw data stored for org: {orgId}")
-         #  Pass RAW PAYLOAD to worker (not just ID)
+       
         task = {
             "id": f"detect_entity:{org_id}:{source_id}:{int(datetime.now().timestamp())}",
             "function": "detect_entity",

app/tasks/worker.py

@@ -31,10 +31,12 @@ def process_detect_entity(org_id: str, **args):
         # 1. ✅ Query DuckDB for latest raw rows (the ones just uploaded)
         conn = get_duckdb(org_id)
         rows = conn.execute("""
-            SELECT row_data FROM main.raw_rows 
-            WHERE row_data IS NOT NULL 
-            ORDER BY _ingested_at DESC LIMIT 500
+             SELECT row_data
+            FROM main.raw_rows
+            WHERE row_data IS NOT NULL
+            USING SAMPLE 40
         """).fetchall()
+
         
         if not rows:
             raise RuntimeError(f"No raw data found for {source_id}")