Spaces:
Sleeping
Sleeping
Commit ·
a73c56c
1
Parent(s): 805478a
fix(BUG-C3,C-03,M-13): wrap synchronous Neo4j session in run_in_executor inside async WebSocket; fix feed change-detection to compare full ID set not just first scraped_at
Browse files- api/main.py +41 -26
api/main.py
CHANGED
|
@@ -169,34 +169,49 @@ async def websocket_feed(websocket: WebSocket):
|
|
| 169 |
payload = {"type": "feed", "at": datetime.now().isoformat()}
|
| 170 |
if driver:
|
| 171 |
try:
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
labels=_FEED_LABELS
|
| 181 |
-
).data()
|
| 182 |
-
current_at = feed_rows[0].get("scraped_at") if feed_rows else None
|
| 183 |
-
# LOGIC FIX: skip push if data has not changed since last send
|
| 184 |
-
if current_at and current_at == last_scraped_at and feed_rows:
|
| 185 |
-
await asyncio.sleep(15)
|
| 186 |
-
continue
|
| 187 |
-
last_scraped_at = current_at
|
| 188 |
-
if feed_rows:
|
| 189 |
-
payload["items"] = feed_rows
|
| 190 |
-
payload["message"] = (
|
| 191 |
-
feed_rows[0].get("label", "Entity") + ": " +
|
| 192 |
-
feed_rows[0].get("name", "-")
|
| 193 |
-
)
|
| 194 |
-
else:
|
| 195 |
rows = s.run(
|
| 196 |
-
"MATCH (n)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 197 |
).data()
|
| 198 |
-
|
| 199 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 200 |
except Exception as db_err:
|
| 201 |
logger.debug(f"[WS] DB query error: {db_err}")
|
| 202 |
payload["message"] = "Feed active -- database query pending"
|
|
|
|
| 169 |
payload = {"type": "feed", "at": datetime.now().isoformat()}
|
| 170 |
if driver:
|
| 171 |
try:
|
| 172 |
+
loop = asyncio.get_event_loop()
|
| 173 |
+
|
| 174 |
+
def _query_feed():
|
| 175 |
+
# C-03 / BUG-C3 FIX: synchronous Neo4j driver MUST run
|
| 176 |
+
# in a thread pool -- calling driver.session() directly
|
| 177 |
+
# inside async def blocks the entire uvicorn event loop
|
| 178 |
+
# while waiting for network I/O to Neo4j AuraDB.
|
| 179 |
+
with driver.session() as s:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
rows = s.run(
|
| 181 |
+
"MATCH (n) WHERE labels(n)[0] IN $labels "
|
| 182 |
+
"AND n.scraped_at IS NOT NULL "
|
| 183 |
+
"RETURN labels(n)[0] AS label, "
|
| 184 |
+
"coalesce(n.name, n.title, n.company_name, n.id) AS name, "
|
| 185 |
+
"n.scraped_at AS scraped_at, n.id AS id, n.source AS source "
|
| 186 |
+
"ORDER BY n.scraped_at DESC LIMIT 8",
|
| 187 |
+
labels=_FEED_LABELS
|
| 188 |
).data()
|
| 189 |
+
if not rows:
|
| 190 |
+
stats = s.run(
|
| 191 |
+
"MATCH (n) RETURN labels(n)[0] AS t, count(n) AS c"
|
| 192 |
+
).data()
|
| 193 |
+
return {"rows": [], "stats": {r["t"]: r["c"] for r in stats if r["t"]}}
|
| 194 |
+
return {"rows": rows, "stats": {}}
|
| 195 |
+
|
| 196 |
+
feed_result = await loop.run_in_executor(None, _query_feed)
|
| 197 |
+
feed_rows = feed_result["rows"]
|
| 198 |
+
|
| 199 |
+
# M-13 FIX: compare full set of IDs, not just first scraped_at
|
| 200 |
+
current_ids = frozenset(r.get("id","") for r in feed_rows)
|
| 201 |
+
if current_ids and current_ids == last_scraped_at:
|
| 202 |
+
await asyncio.sleep(15)
|
| 203 |
+
continue
|
| 204 |
+
last_scraped_at = current_ids
|
| 205 |
+
|
| 206 |
+
if feed_rows:
|
| 207 |
+
payload["items"] = feed_rows
|
| 208 |
+
payload["message"] = (
|
| 209 |
+
feed_rows[0].get("label", "Entity") + ": " +
|
| 210 |
+
feed_rows[0].get("name", "-")
|
| 211 |
+
)
|
| 212 |
+
else:
|
| 213 |
+
payload["stats"] = feed_result["stats"]
|
| 214 |
+
payload["message"] = "Feed active -- run /admin/pipeline to ingest data"
|
| 215 |
except Exception as db_err:
|
| 216 |
logger.debug(f"[WS] DB query error: {db_err}")
|
| 217 |
payload["message"] = "Feed active -- database query pending"
|