abinazebinoy commited on
Commit
a73c56c
·
1 Parent(s): 805478a

fix(BUG-C3,C-03,M-13): wrap synchronous Neo4j session in run_in_executor inside async WebSocket; fix feed change-detection to compare full ID set not just first scraped_at

Browse files
Files changed (1) hide show
  1. api/main.py +41 -26
api/main.py CHANGED
@@ -169,34 +169,49 @@ async def websocket_feed(websocket: WebSocket):
169
  payload = {"type": "feed", "at": datetime.now().isoformat()}
170
  if driver:
171
  try:
172
- with driver.session() as s:
173
- feed_rows = s.run(
174
- "MATCH (n) WHERE labels(n)[0] IN $labels "
175
- "AND n.scraped_at IS NOT NULL "
176
- "RETURN labels(n)[0] AS label, "
177
- "coalesce(n.name, n.title, n.company_name, n.id) AS name, "
178
- "n.scraped_at AS scraped_at, n.id AS id, n.source AS source "
179
- "ORDER BY n.scraped_at DESC LIMIT 8",
180
- labels=_FEED_LABELS
181
- ).data()
182
- current_at = feed_rows[0].get("scraped_at") if feed_rows else None
183
- # LOGIC FIX: skip push if data has not changed since last send
184
- if current_at and current_at == last_scraped_at and feed_rows:
185
- await asyncio.sleep(15)
186
- continue
187
- last_scraped_at = current_at
188
- if feed_rows:
189
- payload["items"] = feed_rows
190
- payload["message"] = (
191
- feed_rows[0].get("label", "Entity") + ": " +
192
- feed_rows[0].get("name", "-")
193
- )
194
- else:
195
  rows = s.run(
196
- "MATCH (n) RETURN labels(n)[0] AS t, count(n) AS c"
 
 
 
 
 
 
197
  ).data()
198
- payload["stats"] = {r["t"]: r["c"] for r in rows if r["t"]}
199
- payload["message"] = "Feed active -- run /admin/pipeline to ingest data"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
200
  except Exception as db_err:
201
  logger.debug(f"[WS] DB query error: {db_err}")
202
  payload["message"] = "Feed active -- database query pending"
 
169
  payload = {"type": "feed", "at": datetime.now().isoformat()}
170
  if driver:
171
  try:
172
+ loop = asyncio.get_event_loop()
173
+
174
+ def _query_feed():
175
+ # C-03 / BUG-C3 FIX: synchronous Neo4j driver MUST run
176
+ # in a thread pool -- calling driver.session() directly
177
+ # inside async def blocks the entire uvicorn event loop
178
+ # while waiting for network I/O to Neo4j AuraDB.
179
+ with driver.session() as s:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  rows = s.run(
181
+ "MATCH (n) WHERE labels(n)[0] IN $labels "
182
+ "AND n.scraped_at IS NOT NULL "
183
+ "RETURN labels(n)[0] AS label, "
184
+ "coalesce(n.name, n.title, n.company_name, n.id) AS name, "
185
+ "n.scraped_at AS scraped_at, n.id AS id, n.source AS source "
186
+ "ORDER BY n.scraped_at DESC LIMIT 8",
187
+ labels=_FEED_LABELS
188
  ).data()
189
+ if not rows:
190
+ stats = s.run(
191
+ "MATCH (n) RETURN labels(n)[0] AS t, count(n) AS c"
192
+ ).data()
193
+ return {"rows": [], "stats": {r["t"]: r["c"] for r in stats if r["t"]}}
194
+ return {"rows": rows, "stats": {}}
195
+
196
+ feed_result = await loop.run_in_executor(None, _query_feed)
197
+ feed_rows = feed_result["rows"]
198
+
199
+ # M-13 FIX: compare full set of IDs, not just first scraped_at
200
+ current_ids = frozenset(r.get("id","") for r in feed_rows)
201
+ if current_ids and current_ids == last_scraped_at:
202
+ await asyncio.sleep(15)
203
+ continue
204
+ last_scraped_at = current_ids
205
+
206
+ if feed_rows:
207
+ payload["items"] = feed_rows
208
+ payload["message"] = (
209
+ feed_rows[0].get("label", "Entity") + ": " +
210
+ feed_rows[0].get("name", "-")
211
+ )
212
+ else:
213
+ payload["stats"] = feed_result["stats"]
214
+ payload["message"] = "Feed active -- run /admin/pipeline to ingest data"
215
  except Exception as db_err:
216
  logger.debug(f"[WS] DB query error: {db_err}")
217
  payload["message"] = "Feed active -- database query pending"