abinazebinoy commited on
Commit
8078bc3
·
1 Parent(s): fabf548

fix(loader): BUG-2/3/17/19/21 — 7 new loaders, fulltext, kwargs, SHA-256

Browse files

BUG-2 (CRITICAL): 7 datasets were scraped but never loaded.
Added loaders: load_icij_entities, load_sanctioned_entities,
load_court_cases, load_local_bodies, load_crime_reports,
load_wikidata_enrichments, load_datagov_documents.
All 7 called in load_from_pipeline_output().

BUG-3: fulltext index in setup_schema() now covers all 20
node types (was only 8). Added: NGO, ElectoralBond,
InsolvencyOrder, Tender, RegulatoryOrder, EnforcementAction,
ParliamentQuestion, VigilanceCircular, ICIJEntity,
SanctionedEntity, CourtCase, LocalBody.

BUG-17: _run() now accepts BOTH positional dict and **kwargs
so new Phase 28 loaders (which used keyword args) work
without errors.

BUG-19: make_id() upgraded from MD5 to SHA-256 (20 hex chars)
for lower collision risk across large datasets.

BUG-21: GraphLoader.__init__ now accepts driver= kwarg so
admin.py can pass the existing FastAPI driver instead of
opening a second Neo4j connection.

Files changed (1) hide show
  1. graph/loader.py +435 -279
graph/loader.py CHANGED
@@ -1,56 +1,43 @@
1
  """
2
  BharatGraph - Neo4j Graph Loader
3
  Loads cleaned + resolved data from Phase 2 pipeline into Neo4j.
4
-
5
- What this does:
6
- 1. Connects to Neo4j AuraDB (free tier)
7
- 2. Creates constraints and indexes (from schema.py)
8
- 3. Loads each entity type as nodes
9
- 4. Creates relationships between connected entities
10
- 5. Reports stats: nodes created, relationships created
11
-
12
- Usage:
13
- python -m graph.loader
14
- python -m graph.loader --dry-run (shows what would be loaded)
15
  """
16
 
17
- import os
18
- import sys
19
- import json
20
- import hashlib
21
- import argparse
22
  from datetime import datetime
23
  from loguru import logger
24
 
25
  sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
26
-
27
  from graph.schema import SETUP_QUERIES, NODE_SCHEMAS
28
 
29
 
 
 
30
  def make_id(*parts) -> str:
31
- """Create a stable ID by hashing multiple string parts."""
32
  combined = "|".join(str(p).lower().strip() for p in parts)
33
- return hashlib.md5(combined.encode()).hexdigest()[:16]
34
 
35
 
36
  class GraphLoader:
37
  """
38
  Loads BharatGraph data into Neo4j.
39
- Handles connection, node creation, and relationship creation.
 
40
  """
41
 
42
- def __init__(self, dry_run: bool = False):
43
  self.dry_run = dry_run
44
- self.driver = None
45
  self.stats = {
46
- "nodes_created": 0,
47
- "nodes_merged": 0,
48
- "rels_created": 0,
49
- "errors": 0,
50
  }
51
  if dry_run:
52
- logger.info("[Loader] DRY RUN mode - no data written to Neo4j")
53
- else:
54
  self._connect()
55
 
56
  def _connect(self):
@@ -61,7 +48,7 @@ class GraphLoader:
61
  load_dotenv()
62
 
63
  uri = os.getenv("NEO4J_URI", "")
64
- user = os.getenv("NEO4J_USER", "neo4j")
65
  pwd = os.getenv("NEO4J_PASSWORD", "")
66
 
67
  if not pwd:
@@ -71,9 +58,8 @@ class GraphLoader:
71
  self.driver = GraphDatabase.driver(uri, auth=(user, pwd))
72
  self.driver.verify_connectivity()
73
  logger.success(f"[Loader] Connected to Neo4j: {uri[:40]}...")
74
-
75
  except ImportError:
76
- logger.error("[Loader] neo4j package not installed. Run: pip install neo4j")
77
  raise
78
  except Exception as e:
79
  logger.error(f"[Loader] Connection failed: {e}")
@@ -85,22 +71,27 @@ class GraphLoader:
85
  logger.info(f"[Loader] DRY RUN: would run {len(SETUP_QUERIES)} setup queries")
86
  return
87
  with self.driver.session() as session:
88
- # Try fulltext index silent fail if already exists
89
  try:
90
  session.run(
91
  "CALL db.index.fulltext.createNodeIndex("
92
  "'globalSearch',"
93
  "['Politician','Company','Contract','AuditReport',"
94
- " 'Scheme','Ministry','Party','PressRelease'],"
 
 
 
 
95
  "['name','title','aliases','item_desc','product','buyer_org',"
96
- " 'cin','ministry','summary','seller_name']"
 
 
97
  ")"
98
  )
99
- logger.info("[Loader] Full-text index created or verified")
100
  except Exception as e:
101
  logger.debug(f"[Loader] Full-text index note: {e}")
102
 
103
- # Run all constraint/index setup queries inside the same session
104
  for query in SETUP_QUERIES:
105
  try:
106
  session.run(query)
@@ -109,15 +100,22 @@ class GraphLoader:
109
 
110
  logger.success("[Loader] Schema constraints and indexes ready")
111
 
112
- def _run(self, query: str, params: dict = None):
113
- """Execute a single Cypher query."""
 
 
 
 
 
 
114
  if self.dry_run:
115
  logger.debug(f"[Loader] DRY RUN query: {query[:80]}...")
116
  return None
 
 
117
  try:
118
  with self.driver.session() as session:
119
- result = session.run(query, params or {})
120
- return result
121
  except Exception as e:
122
  logger.error(f"[Loader] Query failed: {e}\nQuery: {query[:100]}")
123
  self.stats["errors"] += 1
@@ -126,7 +124,6 @@ class GraphLoader:
126
  # ── Node loaders ──────────────────────────────────────
127
 
128
  def load_politicians(self, records: list) -> int:
129
- """Load politician records as (Politician) nodes."""
130
  count = 0
131
  for r in records:
132
  name = r.get("name", "").strip()
@@ -163,8 +160,6 @@ class GraphLoader:
163
  }
164
  self._run(query, params)
165
  count += 1
166
-
167
- # Create Party node + MEMBER_OF relationship
168
  party = r.get("party", "").strip()
169
  if party:
170
  self._create_party_and_link(node_id, party)
@@ -174,7 +169,6 @@ class GraphLoader:
174
  return count
175
 
176
  def _create_party_and_link(self, politician_id: str, party_name: str):
177
- """Create Party node and link politician to it."""
178
  party_id = make_id(party_name)
179
  query = """
180
  MERGE (party:Party {id: $party_id})
@@ -184,13 +178,12 @@ class GraphLoader:
184
  MERGE (p)-[:MEMBER_OF]->(party)
185
  """
186
  self._run(query, {
187
- "party_id": party_id,
188
  "party_name": party_name,
189
- "pol_id": politician_id,
190
  })
191
 
192
  def load_companies(self, records: list) -> int:
193
- """Load company records as (Company) nodes."""
194
  count = 0
195
  for r in records:
196
  name = r.get("name", "").strip()
@@ -229,7 +222,6 @@ class GraphLoader:
229
  return count
230
 
231
  def load_contracts(self, records: list) -> int:
232
- """Load GeM contract records as (Contract) nodes + relationships."""
233
  count = 0
234
  for r in records:
235
  order_id = r.get("order_id", "").strip()
@@ -237,7 +229,6 @@ class GraphLoader:
237
  if not order_id:
238
  continue
239
  node_id = make_id(order_id)
240
- # Create Contract node
241
  query = """
242
  MERGE (c:Contract {id: $id})
243
  SET c.order_id = $order_id,
@@ -264,13 +255,8 @@ class GraphLoader:
264
  }
265
  self._run(query, params)
266
  count += 1
267
-
268
- # Link seller company to this contract
269
  if seller:
270
- self._link_company_to_contract(seller, node_id,
271
- float(r.get("amount_crore", 0) or 0))
272
-
273
- # Link contract to ministry
274
  buyer = r.get("buyer_org", "").strip()
275
  if buyer:
276
  self._link_contract_to_ministry(node_id, buyer)
@@ -279,65 +265,37 @@ class GraphLoader:
279
  self.stats["nodes_created"] += count
280
  return count
281
 
282
- def _link_company_to_contract(self, seller_name: str,
283
- contract_id: str, amount: float):
284
- """Create WON_CONTRACT relationship from Company to Contract."""
285
  company_id = make_id(seller_name)
286
- query = """
287
  MERGE (co:Company {id: $co_id})
288
  SET co.name = $seller_name
289
  WITH co
290
  MATCH (c:Contract {id: $contract_id})
291
  MERGE (co)-[r:WON_CONTRACT]->(c)
292
  SET r.amount_crore = $amount
293
- """
294
- self._run(query, {
295
- "co_id": company_id,
296
- "seller_name": seller_name,
297
- "contract_id": contract_id,
298
- "amount": amount,
299
- })
300
  self.stats["rels_created"] += 1
301
 
302
- def _link_contract_to_ministry(self, contract_id: str, ministry_name: str):
303
- """Create AWARDED_BY relationship from Contract to Ministry."""
304
  ministry_id = make_id(ministry_name)
305
- query = """
306
  MERGE (m:Ministry {id: $m_id})
307
  SET m.name = $m_name
308
  WITH m
309
  MATCH (c:Contract {id: $contract_id})
310
  MERGE (c)-[:AWARDED_BY]->(m)
311
- """
312
- self._run(query, {
313
- "m_id": ministry_id,
314
- "m_name": ministry_name,
315
- "contract_id": contract_id,
316
- })
317
  self.stats["rels_created"] += 1
318
 
319
  def load_audit_reports(self, records: list) -> int:
320
- """Load CAG audit report records as (AuditReport) nodes."""
321
  count = 0
322
  for r in records:
323
  title = r.get("title", "").strip()
324
  if not title:
325
  continue
326
  node_id = make_id(title, r.get("url", ""))
327
- query = """
328
- MERGE (a:AuditReport {id: $id})
329
- SET a.title = $title,
330
- a.url = $url,
331
- a.year = $year,
332
- a.state = $state,
333
- a.scheme = $scheme,
334
- a.amount_crore = $amount,
335
- a.irregularity_type = $irr_type,
336
- a.finding = $finding,
337
- a.alert_keywords = $keywords,
338
- a.source = $source,
339
- a.scraped_at = $scraped_at
340
- """
341
  params = {
342
  "id": node_id,
343
  "title": title,
@@ -352,66 +310,52 @@ class GraphLoader:
352
  "source": r.get("_source", "cag"),
353
  "scraped_at": r.get("scraped_at", datetime.now().isoformat()),
354
  }
355
- self._run(query, params)
 
 
 
 
 
 
356
  count += 1
357
-
358
- # Link to scheme if present
359
  scheme = r.get("scheme", "").strip()
360
  if scheme:
361
- self._link_audit_to_scheme(node_id, scheme,
362
- float(r.get("amount_crore", 0) or 0))
363
 
364
  logger.success(f"[Loader] Audit reports loaded: {count}")
365
  self.stats["nodes_created"] += count
366
  return count
367
 
368
- def _link_audit_to_scheme(self, audit_id: str, scheme_name: str,
369
- amount: float):
370
- """Create FLAGS relationship from AuditReport to Scheme."""
371
  scheme_id = make_id(scheme_name)
372
- query = """
373
  MERGE (s:Scheme {id: $s_id})
374
  SET s.name = $s_name
375
  WITH s
376
  MATCH (a:AuditReport {id: $audit_id})
377
  MERGE (a)-[r:FLAGS]->(s)
378
  SET r.amount_crore = $amount
379
- """
380
- self._run(query, {
381
- "s_id": scheme_id,
382
- "s_name": scheme_name,
383
- "audit_id": audit_id,
384
- "amount": amount,
385
- })
386
  self.stats["rels_created"] += 1
387
 
388
  def load_press_releases(self, records: list) -> int:
389
- """Load PIB press release records as (PressRelease) nodes."""
390
  count = 0
391
  for r in records:
392
  title = r.get("title", "").strip()
393
  if not title:
394
  continue
395
  node_id = make_id(title, r.get("link", ""))
396
- query = """
397
  MERGE (pr:PressRelease {id: $id})
398
- SET pr.title = $title,
399
- pr.link = $link,
400
- pr.published = $published,
401
- pr.alert_keywords = $keywords,
402
- pr.source = $source,
403
- pr.scraped_at = $scraped_at
404
- """
405
- params = {
406
- "id": node_id,
407
- "title": title,
408
- "link": r.get("link", ""),
409
- "published": r.get("published", ""),
410
- "keywords": str(r.get("alert_keywords", [])),
411
- "source": r.get("_source", "pib"),
412
- "scraped_at": r.get("scraped_at", datetime.now().isoformat()),
413
- }
414
- self._run(query, params)
415
  count += 1
416
 
417
  logger.success(f"[Loader] Press releases loaded: {count}")
@@ -419,14 +363,6 @@ class GraphLoader:
419
  return count
420
 
421
  def load_politician_company_links(self, matches: list) -> int:
422
- """
423
- Create DIRECTOR_OF relationships for politician-company matches.
424
- BUG-17 FIX: was using make_id(name) which doesn't match canonical IDs
425
- created by load_politicians() (which uses make_id(name, state, election)).
426
- Fix: MATCH by name using toLower() to find existing nodes, only MERGE the
427
- relationship. If no existing node found, create one with name-only ID as
428
- fallback so the edge is never silently lost.
429
- """
430
  count = 0
431
  for match in matches:
432
  pol_name = match.get("name_a", match.get("politician_name", "")).strip()
@@ -434,26 +370,6 @@ class GraphLoader:
434
  score = match.get("score", 0.0)
435
  if not pol_name or not co_name:
436
  continue
437
-
438
- # BUG-17 FIX: match by name (case-insensitive) to find the canonical node,
439
- # fall back to make_id(name) only if not found in graph yet.
440
- query = """
441
- MERGE (p:Politician {id: coalesce(
442
- (MATCH (x:Politician) WHERE toLower(x.name) = toLower($pol_name) RETURN x.id LIMIT 1)[0],
443
- $pol_id_fallback
444
- )})
445
- SET p.name = coalesce(p.name, $pol_name)
446
- MERGE (c:Company {id: coalesce(
447
- (MATCH (x:Company) WHERE toLower(x.name) = toLower($co_name) RETURN x.id LIMIT 1)[0],
448
- $co_id_fallback
449
- )})
450
- SET c.name = coalesce(c.name, $co_name)
451
- MERGE (p)-[r:DIRECTOR_OF]->(c)
452
- SET r.confidence = $score,
453
- r.source = 'entity_resolution',
454
- r.detected_at = $now
455
- """
456
- # Simpler, more reliable approach using two separate lookups
457
  link_query = """
458
  OPTIONAL MATCH (existing_p:Politician)
459
  WHERE toLower(existing_p.name) = toLower($pol_name)
@@ -489,10 +405,7 @@ class GraphLoader:
489
  return count
490
 
491
  def load_from_pipeline_output(self, filepath: str) -> dict:
492
- """
493
- Load everything from a pipeline JSON output file.
494
- This is the main entry point.
495
- """
496
  logger.info(f"[Loader] Loading from: {filepath}")
497
  with open(filepath, encoding="utf-8") as f:
498
  data = json.load(f)
@@ -503,44 +416,36 @@ class GraphLoader:
503
  self.setup_schema()
504
 
505
  results = {}
506
- if raw.get("myneta"):
507
- results["politicians"] = self.load_politicians(raw["myneta"])
508
- if raw.get("mca"):
509
- results["companies"] = self.load_companies(raw["mca"])
510
- if raw.get("gem"):
511
- results["contracts"] = self.load_contracts(raw["gem"])
512
- if raw.get("cag"):
513
- results["audit_reports"] = self.load_audit_reports(raw["cag"])
514
- if raw.get("pib"):
515
- results["press_releases"] = self.load_press_releases(raw["pib"])
516
- if links:
517
- results["director_of_links"] = self.load_politician_company_links(links)
518
- if raw.get("sebi"):
519
- results["regulatory_orders"] = self.load_regulatory_orders(raw["sebi"])
520
- if raw.get("ed"):
521
- results["enforcement_actions"] = self.load_enforcement_actions(raw["ed"])
522
- if raw.get("electoral_bond"):
523
- results["electoral_bonds"] = self.load_electoral_bonds(raw["electoral_bond"])
524
- if raw.get("ibbi"):
525
- results["insolvency_orders"] = self.load_insolvency_orders(raw["ibbi"])
526
- if raw.get("ngo_darpan"):
527
- results["ngos"] = self.load_ngos(raw["ngo_darpan"])
528
- if raw.get("cppp"):
529
- results["tenders"] = self.load_tenders(raw["cppp"])
530
- if raw.get("loksabha"):
531
- results["parliament_questions"] = self.load_parliament_questions(raw["loksabha"])
532
- if raw.get("cvc"):
533
- results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
534
-
535
 
536
  logger.success(f"[Loader] Load complete. Stats: {self.stats}")
537
  return {**results, "stats": self.stats}
538
 
539
-
540
- # ── Phase 28: 8 new dataset loaders ──────────────────────────────────────
541
 
542
  def load_regulatory_orders(self, records: list) -> int:
543
- """SEBI enforcement orders → RegulatoryOrder nodes."""
544
  count = 0
545
  for r in records:
546
  title = (r.get("title") or "").strip()
@@ -551,12 +456,15 @@ class GraphLoader:
551
  MERGE (n:RegulatoryOrder {id:$id})
552
  SET n.title=$title, n.url=$url, n.order_type=$order_type,
553
  n.entity_name=$entity_name, n.violation=$violation,
554
- n.source=$source, n.dataset="sebi", n.scraped_at=$scraped_at
555
- """, id=node_id, title=title, url=r.get("url",""),
556
- order_type=r.get("order_type",""),
557
- entity_name=r.get("entity_name",r.get("accused","")),
558
- violation=r.get("violation",""), source=r.get("source","SEBI"),
559
- scraped_at=r.get("scraped_at",""))
 
 
 
560
  count += 1
561
  except Exception as e:
562
  logger.warning(f"[Loader] RegulatoryOrder failed: {e}")
@@ -565,7 +473,6 @@ class GraphLoader:
565
  return count
566
 
567
  def load_enforcement_actions(self, records: list) -> int:
568
- """ED enforcement actions → EnforcementAction nodes."""
569
  count = 0
570
  for r in records:
571
  title = (r.get("title") or "").strip()
@@ -577,12 +484,16 @@ class GraphLoader:
577
  SET n.title=$title, n.url=$url, n.date=$date,
578
  n.amount_crore=$amount_crore, n.case_type=$case_type,
579
  n.accused=$accused, n.source=$source,
580
- n.dataset="ed", n.scraped_at=$scraped_at
581
- """, id=node_id, title=title, url=r.get("url",""),
582
- date=r.get("date",""),
583
- amount_crore=float(r.get("amount_crore",0) or 0),
584
- case_type=r.get("case_type",""), accused=r.get("accused",""),
585
- source=r.get("source","ED"), scraped_at=r.get("scraped_at",""))
 
 
 
 
586
  count += 1
587
  accused = r.get("accused","").strip()
588
  if accused:
@@ -591,7 +502,7 @@ class GraphLoader:
591
  WHERE toLower(p.name) CONTAINS toLower($accused)
592
  MATCH (n:EnforcementAction {id:$id})
593
  MERGE (p)-[:SUBJECT_OF]->(n)
594
- """, accused=accused, id=node_id)
595
  except Exception as e:
596
  logger.warning(f"[Loader] EnforcementAction failed: {e}")
597
  self.stats["enforcement_actions"] = count
@@ -599,7 +510,6 @@ class GraphLoader:
599
  return count
600
 
601
  def load_electoral_bonds(self, records: list) -> int:
602
- """Electoral bonds → ElectoralBond nodes + DONATED_VIA/REDEEMED_BY."""
603
  count = 0
604
  for r in records:
605
  purchaser = (r.get("purchaser_name") or "").strip()
@@ -611,26 +521,33 @@ class GraphLoader:
611
  SET n.bond_number=$bond_no, n.purchaser_name=$purchaser,
612
  n.denomination_crore=$denom, n.purchase_date=$p_date,
613
  n.redemption_date=$r_date, n.redeemed_by=$redeemer,
614
- n.source=$source, n.dataset="electoral_bond",
615
  n.scraped_at=$scraped_at
616
- """, id=node_id, bond_no=r.get("bond_number",""),
617
- purchaser=purchaser,
618
- denom=float(r.get("denomination_crore",0) or 0),
619
- p_date=r.get("purchase_date",""), r_date=r.get("redemption_date",""),
620
- redeemer=r.get("redeemed_by",""), source=r.get("source","ECI"),
621
- scraped_at=r.get("scraped_at",""))
 
 
 
 
622
  count += 1
623
  redeemer = r.get("redeemed_by","").strip()
624
  if purchaser and redeemer:
625
  self._run("""
626
  MERGE (c:Company {id:$co_id})
627
- ON CREATE SET c.name=$purchaser, c.source="electoral_bond"
628
  MERGE (p:Party {id:$pt_id})
629
- ON CREATE SET p.name=$redeemer, p.source="electoral_bond"
630
  MATCH (b:ElectoralBond {id:$b_id})
631
  MERGE (c)-[:DONATED_VIA]->(b)-[:REDEEMED_BY]->(p)
632
- """, co_id=make_id(purchaser), purchaser=purchaser,
633
- pt_id=make_id(redeemer), redeemer=redeemer, b_id=node_id)
 
 
 
634
  except Exception as e:
635
  logger.warning(f"[Loader] ElectoralBond failed: {e}")
636
  self.stats["electoral_bonds"] = count
@@ -638,7 +555,6 @@ class GraphLoader:
638
  return count
639
 
640
  def load_insolvency_orders(self, records: list) -> int:
641
- """IBBI insolvency orders → InsolvencyOrder + HAS_INSOLVENCY link."""
642
  count = 0
643
  for r in records:
644
  company = (r.get("company_name") or "").strip()
@@ -650,19 +566,23 @@ class GraphLoader:
650
  SET n.company_name=$company, n.cin=$cin,
651
  n.process_type=$process_type, n.admitted_date=$admitted_date,
652
  n.status=$status, n.admitted_claims=$claims,
653
- n.source=$source, n.dataset="ibbi", n.scraped_at=$scraped_at
654
- """, id=node_id, company=company, cin=r.get("cin",""),
655
- process_type=r.get("process_type",""),
656
- admitted_date=r.get("admitted_date",""), status=r.get("status",""),
657
- claims=float(r.get("admitted_claims",0) or 0),
658
- source=r.get("source","IBBI"), scraped_at=r.get("scraped_at",""))
 
 
 
 
659
  count += 1
660
  self._run("""
661
  MATCH (c:Company)
662
  WHERE toLower(c.name) CONTAINS toLower($company)
663
  MATCH (n:InsolvencyOrder {id:$id})
664
  MERGE (c)-[:HAS_INSOLVENCY]->(n)
665
- """, company=company, id=node_id)
666
  except Exception as e:
667
  logger.warning(f"[Loader] InsolvencyOrder failed: {e}")
668
  self.stats["insolvency_orders"] = count
@@ -670,7 +590,6 @@ class GraphLoader:
670
  return count
671
 
672
  def load_ngos(self, records: list) -> int:
673
- """NGO Darpan → NGO nodes."""
674
  count = 0
675
  for r in records:
676
  name = (r.get("ngo_name") or "").strip()
@@ -684,15 +603,20 @@ class GraphLoader:
684
  n.registration_type=$reg_type, n.year_of_reg=$year,
685
  n.key_issues=$issues, n.csr_receipts=$csr,
686
  n.govt_grants=$grants, n.source=$source,
687
- n.dataset="ngo_darpan", n.scraped_at=$scraped_at
688
- """, id=node_id, name=name, darpan_id=r.get("darpan_id",""),
689
- state=r.get("state",""), district=r.get("district",""),
690
- reg_type=r.get("registration_type",""),
691
- year=str(r.get("year_of_reg","")),
692
- issues=str(r.get("key_issues",[])),
693
- csr=float(r.get("csr_receipts",0) or 0),
694
- grants=float(r.get("govt_grants",0) or 0),
695
- source=r.get("source","NGO Darpan"), scraped_at=r.get("scraped_at",""))
 
 
 
 
 
696
  count += 1
697
  except Exception as e:
698
  logger.warning(f"[Loader] NGO failed: {e}")
@@ -701,7 +625,6 @@ class GraphLoader:
701
  return count
702
 
703
  def load_tenders(self, records: list) -> int:
704
- """CPPP tenders → Tender nodes + ISSUED_TENDER ministry link."""
705
  count = 0
706
  for r in records:
707
  title = (r.get("title") or "").strip()
@@ -716,27 +639,34 @@ class GraphLoader:
716
  n.bid_end_date=$bid_end, n.status=$status,
717
  n.awarded_to=$awarded_to, n.awarded_value=$awarded_val,
718
  n.single_bid=$single_bid, n.source=$source,
719
- n.dataset="cppp", n.scraped_at=$scraped_at
720
- """, id=node_id, tender_id=r.get("tender_id",""), title=title,
721
- ministry=r.get("ministry",""), dept=r.get("department",""),
722
- value=float(r.get("estimated_crore",0) or 0),
723
- t_type=r.get("tender_type",""), bid_end=r.get("bid_submission_end",""),
724
- status=r.get("status",""), awarded_to=r.get("awarded_to",""),
725
- awarded_val=float(r.get("awarded_value",0) or 0),
726
- single_bid=bool(r.get("single_bid",False)),
727
- source=r.get("source","CPPP"), scraped_at=r.get("scraped_at",""))
 
 
 
 
 
 
 
728
  count += 1
729
  ministry = r.get("ministry","").strip()
730
  if ministry:
731
  self._run("""
732
  MERGE (m:Ministry {id:$m_id})
733
- ON CREATE SET m.name=$ministry, m.source="cppp"
734
  MATCH (t:Tender {id:$t_id})
735
  MERGE (m)-[:ISSUED_TENDER]->(t)
736
- """, m_id=make_id(ministry), ministry=ministry, t_id=node_id)
737
  if r.get("single_bid"):
738
  self._run("MATCH (t:Tender {id:$id}) SET t.risk_flag='SINGLE_BID'",
739
- id=node_id)
740
  except Exception as e:
741
  logger.warning(f"[Loader] Tender failed: {e}")
742
  self.stats["tenders"] = count
@@ -744,7 +674,6 @@ class GraphLoader:
744
  return count
745
 
746
  def load_parliament_questions(self, records: list) -> int:
747
- """Lok Sabha questions → ParliamentQuestion nodes."""
748
  count = 0
749
  for r in records:
750
  subject = (r.get("subject") or "").strip()
@@ -756,11 +685,16 @@ class GraphLoader:
756
  SET n.subject=$subject, n.question_number=$qno,
757
  n.member_name=$member, n.ministry=$ministry,
758
  n.session=$session, n.source=$source,
759
- n.dataset="loksabha", n.scraped_at=$scraped_at
760
- """, id=node_id, subject=subject,
761
- qno=r.get("question_number",""), member=r.get("member_name",""),
762
- ministry=r.get("ministry",""), session=r.get("session",""),
763
- source=r.get("source","Lok Sabha"), scraped_at=r.get("scraped_at",""))
 
 
 
 
 
764
  count += 1
765
  member = r.get("member_name","").strip()
766
  if member:
@@ -769,7 +703,7 @@ class GraphLoader:
769
  WHERE toLower(p.name) CONTAINS toLower($member)
770
  MATCH (q:ParliamentQuestion {id:$id})
771
  MERGE (p)-[:ASKED_QUESTION]->(q)
772
- """, member=member, id=node_id)
773
  except Exception as e:
774
  logger.warning(f"[Loader] ParliamentQuestion failed: {e}")
775
  self.stats["parliament_questions"] = count
@@ -777,7 +711,6 @@ class GraphLoader:
777
  return count
778
 
779
  def load_vigilance_circulars(self, records: list) -> int:
780
- """CVC circulars → VigilanceCircular nodes."""
781
  count = 0
782
  for r in records:
783
  title = (r.get("title") or "").strip()
@@ -789,11 +722,16 @@ class GraphLoader:
789
  SET n.title=$title, n.circular_number=$cno,
790
  n.date=$date, n.ministry=$ministry,
791
  n.subject=$subject, n.source=$source,
792
- n.dataset="cvc", n.scraped_at=$scraped_at
793
- """, id=node_id, title=title, cno=r.get("circular_number",""),
794
- date=r.get("date",""), ministry=r.get("ministry",""),
795
- subject=r.get("subject",""), source=r.get("source","CVC"),
796
- scraped_at=r.get("scraped_at",""))
 
 
 
 
 
797
  count += 1
798
  except Exception as e:
799
  logger.warning(f"[Loader] VigilanceCircular failed: {e}")
@@ -801,6 +739,233 @@ class GraphLoader:
801
  logger.success(f"[Loader] Loaded {count} CVC circulars")
802
  return count
803
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
804
  def close(self):
805
  if self.driver:
806
  self.driver.close()
@@ -809,10 +974,8 @@ class GraphLoader:
809
 
810
  if __name__ == "__main__":
811
  parser = argparse.ArgumentParser(description="BharatGraph Graph Loader")
812
- parser.add_argument("--dry-run", action="store_true",
813
- help="Show what would be loaded without writing to Neo4j")
814
- parser.add_argument("--file", type=str, default=None,
815
- help="Path to pipeline output JSON file")
816
  args = parser.parse_args()
817
 
818
  print("=" * 55)
@@ -824,28 +987,21 @@ if __name__ == "__main__":
824
  if args.file:
825
  results = loader.load_from_pipeline_output(args.file)
826
  else:
827
- # Find latest pipeline output
828
  import glob
829
  files = sorted(glob.glob("data/processed/pipeline_*.json"))
830
  if files:
831
- latest = files[-1]
832
- print(f"\nLoading latest pipeline output: {latest}")
833
- results = loader.load_from_pipeline_output(latest)
834
  else:
835
- print("\nNo pipeline output found.")
836
- print("Run first: python -m processing.pipeline --scrapers cag,gem,pib")
837
  results = {}
838
 
839
  loader.close()
840
-
841
- print(f"\n{'='*55}")
842
- print("LOAD SUMMARY")
843
- print(f"{'='*55}")
844
  s = loader.stats
845
- print(f" Nodes created: {s['nodes_created']}")
846
- print(f" Nodes merged: {s['nodes_merged']}")
847
- print(f" Relationships: {s['rels_created']}")
848
- print(f" Errors: {s['errors']}")
 
849
  if args.dry_run:
850
- print("\n (DRY RUN - nothing was written to Neo4j)")
851
- print(f"{'='*55}")
 
1
  """
2
  BharatGraph - Neo4j Graph Loader
3
  Loads cleaned + resolved data from Phase 2 pipeline into Neo4j.
 
 
 
 
 
 
 
 
 
 
 
4
  """
5
 
6
+ import os, sys, json, hashlib, argparse
 
 
 
 
7
  from datetime import datetime
8
  from loguru import logger
9
 
10
  sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
11
  from graph.schema import SETUP_QUERIES, NODE_SCHEMAS
12
 
13
 
14
+ # BUG-19 FIX: upgraded from MD5 (16-char) to SHA-256 (20-char) for lower collision
15
+ # risk across large datasets.
16
  def make_id(*parts) -> str:
17
+ """Create a stable SHA-256 derived ID from multiple string parts."""
18
  combined = "|".join(str(p).lower().strip() for p in parts)
19
+ return hashlib.sha256(combined.encode()).hexdigest()[:20]
20
 
21
 
22
  class GraphLoader:
23
  """
24
  Loads BharatGraph data into Neo4j.
25
+ BUG-21 FIX: __init__ now accepts an optional pre-built driver so admin.py
26
+ can pass the FastAPI dependency driver instead of opening a second connection.
27
  """
28
 
29
+ def __init__(self, dry_run: bool = False, driver=None):
30
  self.dry_run = dry_run
31
+ self.driver = driver # BUG-21 FIX: accept injected driver
32
  self.stats = {
33
+ "nodes_created": 0,
34
+ "nodes_merged": 0,
35
+ "rels_created": 0,
36
+ "errors": 0,
37
  }
38
  if dry_run:
39
+ logger.info("[Loader] DRY RUN mode no data written to Neo4j")
40
+ elif driver is None:
41
  self._connect()
42
 
43
  def _connect(self):
 
48
  load_dotenv()
49
 
50
  uri = os.getenv("NEO4J_URI", "")
51
+ user = os.getenv("NEO4J_USER", "neo4j")
52
  pwd = os.getenv("NEO4J_PASSWORD", "")
53
 
54
  if not pwd:
 
58
  self.driver = GraphDatabase.driver(uri, auth=(user, pwd))
59
  self.driver.verify_connectivity()
60
  logger.success(f"[Loader] Connected to Neo4j: {uri[:40]}...")
 
61
  except ImportError:
62
+ logger.error("[Loader] neo4j package not installed.")
63
  raise
64
  except Exception as e:
65
  logger.error(f"[Loader] Connection failed: {e}")
 
71
  logger.info(f"[Loader] DRY RUN: would run {len(SETUP_QUERIES)} setup queries")
72
  return
73
  with self.driver.session() as session:
74
+ # BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
75
  try:
76
  session.run(
77
  "CALL db.index.fulltext.createNodeIndex("
78
  "'globalSearch',"
79
  "['Politician','Company','Contract','AuditReport',"
80
+ " 'Scheme','Ministry','Party','PressRelease',"
81
+ " 'NGO','ElectoralBond','InsolvencyOrder','Tender',"
82
+ " 'RegulatoryOrder','EnforcementAction',"
83
+ " 'ParliamentQuestion','VigilanceCircular',"
84
+ " 'ICIJEntity','SanctionedEntity','CourtCase','LocalBody'],"
85
  "['name','title','aliases','item_desc','product','buyer_org',"
86
+ " 'cin','ministry','summary','seller_name','ngo_name',"
87
+ " 'purchaser_name','redeemed_by','company_name','accused',"
88
+ " 'entity_name','subject','jurisdiction']"
89
  ")"
90
  )
91
+ logger.info("[Loader] Full-text index created or verified (20 types)")
92
  except Exception as e:
93
  logger.debug(f"[Loader] Full-text index note: {e}")
94
 
 
95
  for query in SETUP_QUERIES:
96
  try:
97
  session.run(query)
 
100
 
101
  logger.success("[Loader] Schema constraints and indexes ready")
102
 
103
+ def _run(self, query: str, params: dict = None, **kwargs):
104
+ """
105
+ Execute a single Cypher query.
106
+ BUG-17 FIX: now accepts both calling styles:
107
+ - old style: self._run(query, {"key": val}) ← positional dict
108
+ - new style: self._run(query, key=val, ...) ← keyword args
109
+ Both are merged so existing callers and the 8 new loaders both work.
110
+ """
111
  if self.dry_run:
112
  logger.debug(f"[Loader] DRY RUN query: {query[:80]}...")
113
  return None
114
+ # Merge positional dict with any keyword args
115
+ all_params = {**(params or {}), **kwargs}
116
  try:
117
  with self.driver.session() as session:
118
+ return session.run(query, all_params)
 
119
  except Exception as e:
120
  logger.error(f"[Loader] Query failed: {e}\nQuery: {query[:100]}")
121
  self.stats["errors"] += 1
 
124
  # ── Node loaders ──────────────────────────────────────
125
 
126
  def load_politicians(self, records: list) -> int:
 
127
  count = 0
128
  for r in records:
129
  name = r.get("name", "").strip()
 
160
  }
161
  self._run(query, params)
162
  count += 1
 
 
163
  party = r.get("party", "").strip()
164
  if party:
165
  self._create_party_and_link(node_id, party)
 
169
  return count
170
 
171
  def _create_party_and_link(self, politician_id: str, party_name: str):
 
172
  party_id = make_id(party_name)
173
  query = """
174
  MERGE (party:Party {id: $party_id})
 
178
  MERGE (p)-[:MEMBER_OF]->(party)
179
  """
180
  self._run(query, {
181
+ "party_id": party_id,
182
  "party_name": party_name,
183
+ "pol_id": politician_id,
184
  })
185
 
186
  def load_companies(self, records: list) -> int:
 
187
  count = 0
188
  for r in records:
189
  name = r.get("name", "").strip()
 
222
  return count
223
 
224
  def load_contracts(self, records: list) -> int:
 
225
  count = 0
226
  for r in records:
227
  order_id = r.get("order_id", "").strip()
 
229
  if not order_id:
230
  continue
231
  node_id = make_id(order_id)
 
232
  query = """
233
  MERGE (c:Contract {id: $id})
234
  SET c.order_id = $order_id,
 
255
  }
256
  self._run(query, params)
257
  count += 1
 
 
258
  if seller:
259
+ self._link_company_to_contract(seller, node_id, float(r.get("amount_crore", 0) or 0))
 
 
 
260
  buyer = r.get("buyer_org", "").strip()
261
  if buyer:
262
  self._link_contract_to_ministry(node_id, buyer)
 
265
  self.stats["nodes_created"] += count
266
  return count
267
 
268
+ def _link_company_to_contract(self, seller_name, contract_id, amount):
 
 
269
  company_id = make_id(seller_name)
270
+ self._run("""
271
  MERGE (co:Company {id: $co_id})
272
  SET co.name = $seller_name
273
  WITH co
274
  MATCH (c:Contract {id: $contract_id})
275
  MERGE (co)-[r:WON_CONTRACT]->(c)
276
  SET r.amount_crore = $amount
277
+ """, {"co_id": company_id, "seller_name": seller_name,
278
+ "contract_id": contract_id, "amount": amount})
 
 
 
 
 
279
  self.stats["rels_created"] += 1
280
 
281
+ def _link_contract_to_ministry(self, contract_id, ministry_name):
 
282
  ministry_id = make_id(ministry_name)
283
+ self._run("""
284
  MERGE (m:Ministry {id: $m_id})
285
  SET m.name = $m_name
286
  WITH m
287
  MATCH (c:Contract {id: $contract_id})
288
  MERGE (c)-[:AWARDED_BY]->(m)
289
+ """, {"m_id": ministry_id, "m_name": ministry_name, "contract_id": contract_id})
 
 
 
 
 
290
  self.stats["rels_created"] += 1
291
 
292
  def load_audit_reports(self, records: list) -> int:
 
293
  count = 0
294
  for r in records:
295
  title = r.get("title", "").strip()
296
  if not title:
297
  continue
298
  node_id = make_id(title, r.get("url", ""))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
299
  params = {
300
  "id": node_id,
301
  "title": title,
 
310
  "source": r.get("_source", "cag"),
311
  "scraped_at": r.get("scraped_at", datetime.now().isoformat()),
312
  }
313
+ self._run("""
314
+ MERGE (a:AuditReport {id: $id})
315
+ SET a.title=$title, a.url=$url, a.year=$year, a.state=$state,
316
+ a.scheme=$scheme, a.amount_crore=$amount,
317
+ a.irregularity_type=$irr_type, a.finding=$finding,
318
+ a.alert_keywords=$keywords, a.source=$source, a.scraped_at=$scraped_at
319
+ """, params)
320
  count += 1
 
 
321
  scheme = r.get("scheme", "").strip()
322
  if scheme:
323
+ self._link_audit_to_scheme(node_id, scheme, float(r.get("amount_crore", 0) or 0))
 
324
 
325
  logger.success(f"[Loader] Audit reports loaded: {count}")
326
  self.stats["nodes_created"] += count
327
  return count
328
 
329
+ def _link_audit_to_scheme(self, audit_id, scheme_name, amount):
 
 
330
  scheme_id = make_id(scheme_name)
331
+ self._run("""
332
  MERGE (s:Scheme {id: $s_id})
333
  SET s.name = $s_name
334
  WITH s
335
  MATCH (a:AuditReport {id: $audit_id})
336
  MERGE (a)-[r:FLAGS]->(s)
337
  SET r.amount_crore = $amount
338
+ """, {"s_id": scheme_id, "s_name": scheme_name, "audit_id": audit_id, "amount": amount})
 
 
 
 
 
 
339
  self.stats["rels_created"] += 1
340
 
341
  def load_press_releases(self, records: list) -> int:
 
342
  count = 0
343
  for r in records:
344
  title = r.get("title", "").strip()
345
  if not title:
346
  continue
347
  node_id = make_id(title, r.get("link", ""))
348
+ self._run("""
349
  MERGE (pr:PressRelease {id: $id})
350
+ SET pr.title=$title, pr.link=$link, pr.published=$published,
351
+ pr.alert_keywords=$keywords, pr.source=$source, pr.scraped_at=$scraped_at
352
+ """, {
353
+ "id": node_id, "title": title, "link": r.get("link", ""),
354
+ "published": r.get("published", ""),
355
+ "keywords": str(r.get("alert_keywords", [])),
356
+ "source": r.get("_source", "pib"),
357
+ "scraped_at":r.get("scraped_at", datetime.now().isoformat()),
358
+ })
 
 
 
 
 
 
 
 
359
  count += 1
360
 
361
  logger.success(f"[Loader] Press releases loaded: {count}")
 
363
  return count
364
 
365
  def load_politician_company_links(self, matches: list) -> int:
 
 
 
 
 
 
 
 
366
  count = 0
367
  for match in matches:
368
  pol_name = match.get("name_a", match.get("politician_name", "")).strip()
 
370
  score = match.get("score", 0.0)
371
  if not pol_name or not co_name:
372
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
373
  link_query = """
374
  OPTIONAL MATCH (existing_p:Politician)
375
  WHERE toLower(existing_p.name) = toLower($pol_name)
 
405
  return count
406
 
407
  def load_from_pipeline_output(self, filepath: str) -> dict:
408
+ """Load everything from a pipeline JSON output file."""
 
 
 
409
  logger.info(f"[Loader] Loading from: {filepath}")
410
  with open(filepath, encoding="utf-8") as f:
411
  data = json.load(f)
 
416
  self.setup_schema()
417
 
418
  results = {}
419
+ if raw.get("myneta"): results["politicians"] = self.load_politicians(raw["myneta"])
420
+ if raw.get("mca"): results["companies"] = self.load_companies(raw["mca"])
421
+ if raw.get("gem"): results["contracts"] = self.load_contracts(raw["gem"])
422
+ if raw.get("cag"): results["audit_reports"] = self.load_audit_reports(raw["cag"])
423
+ if raw.get("pib"): results["press_releases"] = self.load_press_releases(raw["pib"])
424
+ if links: results["director_of_links"] = self.load_politician_company_links(links)
425
+ if raw.get("sebi"): results["regulatory_orders"] = self.load_regulatory_orders(raw["sebi"])
426
+ if raw.get("ed"): results["enforcement_actions"] = self.load_enforcement_actions(raw["ed"])
427
+ if raw.get("electoral_bond"):results["electoral_bonds"] = self.load_electoral_bonds(raw["electoral_bond"])
428
+ if raw.get("ibbi"): results["insolvency_orders"] = self.load_insolvency_orders(raw["ibbi"])
429
+ if raw.get("ngo_darpan"): results["ngos"] = self.load_ngos(raw["ngo_darpan"])
430
+ if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
431
+ if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
432
+ if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
433
+ # BUG-2 FIX: 7 previously missing loaders — these datasets were scraped but
434
+ # silently dropped because load_from_pipeline_output never called them.
435
+ if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
436
+ if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
437
+ if raw.get("njdg"): results["court_cases"] = self.load_court_cases(raw["njdg"])
438
+ if raw.get("lgd"): results["local_bodies"] = self.load_local_bodies(raw["lgd"])
439
+ if raw.get("ncrb"): results["crime_reports"] = self.load_crime_reports(raw["ncrb"])
440
+ if raw.get("wikidata"): results["wikidata_enrichments"]= self.load_wikidata_enrichments(raw["wikidata"])
441
+ if raw.get("datagov"): results["datagov_documents"] = self.load_datagov_documents(raw["datagov"])
 
 
 
 
 
 
442
 
443
  logger.success(f"[Loader] Load complete. Stats: {self.stats}")
444
  return {**results, "stats": self.stats}
445
 
446
+ # ── Phase 28 loaders (8 datasets) ────────────────────────────────────────
 
447
 
448
  def load_regulatory_orders(self, records: list) -> int:
 
449
  count = 0
450
  for r in records:
451
  title = (r.get("title") or "").strip()
 
456
  MERGE (n:RegulatoryOrder {id:$id})
457
  SET n.title=$title, n.url=$url, n.order_type=$order_type,
458
  n.entity_name=$entity_name, n.violation=$violation,
459
+ n.source=$source, n.dataset='sebi', n.scraped_at=$scraped_at
460
+ """, {
461
+ "id": node_id, "title": title, "url": r.get("url",""),
462
+ "order_type": r.get("order_type",""),
463
+ "entity_name": r.get("entity_name", r.get("accused","")),
464
+ "violation": r.get("violation",""),
465
+ "source": r.get("source","SEBI"),
466
+ "scraped_at": r.get("scraped_at",""),
467
+ })
468
  count += 1
469
  except Exception as e:
470
  logger.warning(f"[Loader] RegulatoryOrder failed: {e}")
 
473
  return count
474
 
475
  def load_enforcement_actions(self, records: list) -> int:
 
476
  count = 0
477
  for r in records:
478
  title = (r.get("title") or "").strip()
 
484
  SET n.title=$title, n.url=$url, n.date=$date,
485
  n.amount_crore=$amount_crore, n.case_type=$case_type,
486
  n.accused=$accused, n.source=$source,
487
+ n.dataset='ed', n.scraped_at=$scraped_at
488
+ """, {
489
+ "id": node_id, "title": title, "url": r.get("url",""),
490
+ "date": r.get("date",""),
491
+ "amount_crore": float(r.get("amount_crore",0) or 0),
492
+ "case_type": r.get("case_type",""),
493
+ "accused": r.get("accused",""),
494
+ "source": r.get("source","ED"),
495
+ "scraped_at": r.get("scraped_at",""),
496
+ })
497
  count += 1
498
  accused = r.get("accused","").strip()
499
  if accused:
 
502
  WHERE toLower(p.name) CONTAINS toLower($accused)
503
  MATCH (n:EnforcementAction {id:$id})
504
  MERGE (p)-[:SUBJECT_OF]->(n)
505
+ """, {"accused": accused, "id": node_id})
506
  except Exception as e:
507
  logger.warning(f"[Loader] EnforcementAction failed: {e}")
508
  self.stats["enforcement_actions"] = count
 
510
  return count
511
 
512
  def load_electoral_bonds(self, records: list) -> int:
 
513
  count = 0
514
  for r in records:
515
  purchaser = (r.get("purchaser_name") or "").strip()
 
521
  SET n.bond_number=$bond_no, n.purchaser_name=$purchaser,
522
  n.denomination_crore=$denom, n.purchase_date=$p_date,
523
  n.redemption_date=$r_date, n.redeemed_by=$redeemer,
524
+ n.source=$source, n.dataset='electoral_bond',
525
  n.scraped_at=$scraped_at
526
+ """, {
527
+ "id": node_id, "bond_no": r.get("bond_number",""),
528
+ "purchaser": purchaser,
529
+ "denom": float(r.get("denomination_crore",0) or 0),
530
+ "p_date": r.get("purchase_date",""),
531
+ "r_date": r.get("redemption_date",""),
532
+ "redeemer": r.get("redeemed_by",""),
533
+ "source": r.get("source","ECI"),
534
+ "scraped_at":r.get("scraped_at",""),
535
+ })
536
  count += 1
537
  redeemer = r.get("redeemed_by","").strip()
538
  if purchaser and redeemer:
539
  self._run("""
540
  MERGE (c:Company {id:$co_id})
541
+ ON CREATE SET c.name=$purchaser, c.source='electoral_bond'
542
  MERGE (p:Party {id:$pt_id})
543
+ ON CREATE SET p.name=$redeemer, p.source='electoral_bond'
544
  MATCH (b:ElectoralBond {id:$b_id})
545
  MERGE (c)-[:DONATED_VIA]->(b)-[:REDEEMED_BY]->(p)
546
+ """, {
547
+ "co_id": make_id(purchaser), "purchaser": purchaser,
548
+ "pt_id": make_id(redeemer), "redeemer": redeemer,
549
+ "b_id": node_id,
550
+ })
551
  except Exception as e:
552
  logger.warning(f"[Loader] ElectoralBond failed: {e}")
553
  self.stats["electoral_bonds"] = count
 
555
  return count
556
 
557
  def load_insolvency_orders(self, records: list) -> int:
 
558
  count = 0
559
  for r in records:
560
  company = (r.get("company_name") or "").strip()
 
566
  SET n.company_name=$company, n.cin=$cin,
567
  n.process_type=$process_type, n.admitted_date=$admitted_date,
568
  n.status=$status, n.admitted_claims=$claims,
569
+ n.source=$source, n.dataset='ibbi', n.scraped_at=$scraped_at
570
+ """, {
571
+ "id": node_id, "company": company, "cin": r.get("cin",""),
572
+ "process_type": r.get("process_type",""),
573
+ "admitted_date": r.get("admitted_date",""),
574
+ "status": r.get("status",""),
575
+ "claims": float(r.get("admitted_claims",0) or 0),
576
+ "source": r.get("source","IBBI"),
577
+ "scraped_at": r.get("scraped_at",""),
578
+ })
579
  count += 1
580
  self._run("""
581
  MATCH (c:Company)
582
  WHERE toLower(c.name) CONTAINS toLower($company)
583
  MATCH (n:InsolvencyOrder {id:$id})
584
  MERGE (c)-[:HAS_INSOLVENCY]->(n)
585
+ """, {"company": company, "id": node_id})
586
  except Exception as e:
587
  logger.warning(f"[Loader] InsolvencyOrder failed: {e}")
588
  self.stats["insolvency_orders"] = count
 
590
  return count
591
 
592
  def load_ngos(self, records: list) -> int:
 
593
  count = 0
594
  for r in records:
595
  name = (r.get("ngo_name") or "").strip()
 
603
  n.registration_type=$reg_type, n.year_of_reg=$year,
604
  n.key_issues=$issues, n.csr_receipts=$csr,
605
  n.govt_grants=$grants, n.source=$source,
606
+ n.dataset='ngo_darpan', n.scraped_at=$scraped_at
607
+ """, {
608
+ "id": node_id, "name": name,
609
+ "darpan_id": r.get("darpan_id",""),
610
+ "state": r.get("state",""),
611
+ "district": r.get("district",""),
612
+ "reg_type": r.get("registration_type",""),
613
+ "year": str(r.get("year_of_reg","")),
614
+ "issues": str(r.get("key_issues",[])),
615
+ "csr": float(r.get("csr_receipts",0) or 0),
616
+ "grants": float(r.get("govt_grants",0) or 0),
617
+ "source": r.get("source","NGO Darpan"),
618
+ "scraped_at":r.get("scraped_at",""),
619
+ })
620
  count += 1
621
  except Exception as e:
622
  logger.warning(f"[Loader] NGO failed: {e}")
 
625
  return count
626
 
627
  def load_tenders(self, records: list) -> int:
 
628
  count = 0
629
  for r in records:
630
  title = (r.get("title") or "").strip()
 
639
  n.bid_end_date=$bid_end, n.status=$status,
640
  n.awarded_to=$awarded_to, n.awarded_value=$awarded_val,
641
  n.single_bid=$single_bid, n.source=$source,
642
+ n.dataset='cppp', n.scraped_at=$scraped_at
643
+ """, {
644
+ "id": node_id, "tender_id": r.get("tender_id",""),
645
+ "title": title,
646
+ "ministry": r.get("ministry",""),
647
+ "dept": r.get("department",""),
648
+ "value": float(r.get("estimated_crore",0) or 0),
649
+ "t_type": r.get("tender_type",""),
650
+ "bid_end": r.get("bid_submission_end",""),
651
+ "status": r.get("status",""),
652
+ "awarded_to": r.get("awarded_to",""),
653
+ "awarded_val":float(r.get("awarded_value",0) or 0),
654
+ "single_bid": bool(r.get("single_bid",False)),
655
+ "source": r.get("source","CPPP"),
656
+ "scraped_at": r.get("scraped_at",""),
657
+ })
658
  count += 1
659
  ministry = r.get("ministry","").strip()
660
  if ministry:
661
  self._run("""
662
  MERGE (m:Ministry {id:$m_id})
663
+ ON CREATE SET m.name=$ministry, m.source='cppp'
664
  MATCH (t:Tender {id:$t_id})
665
  MERGE (m)-[:ISSUED_TENDER]->(t)
666
+ """, {"m_id": make_id(ministry), "ministry": ministry, "t_id": node_id})
667
  if r.get("single_bid"):
668
  self._run("MATCH (t:Tender {id:$id}) SET t.risk_flag='SINGLE_BID'",
669
+ {"id": node_id})
670
  except Exception as e:
671
  logger.warning(f"[Loader] Tender failed: {e}")
672
  self.stats["tenders"] = count
 
674
  return count
675
 
676
  def load_parliament_questions(self, records: list) -> int:
 
677
  count = 0
678
  for r in records:
679
  subject = (r.get("subject") or "").strip()
 
685
  SET n.subject=$subject, n.question_number=$qno,
686
  n.member_name=$member, n.ministry=$ministry,
687
  n.session=$session, n.source=$source,
688
+ n.dataset='loksabha', n.scraped_at=$scraped_at
689
+ """, {
690
+ "id": node_id, "subject": subject,
691
+ "qno": r.get("question_number",""),
692
+ "member": r.get("member_name",""),
693
+ "ministry": r.get("ministry",""),
694
+ "session": r.get("session",""),
695
+ "source": r.get("source","Lok Sabha"),
696
+ "scraped_at":r.get("scraped_at",""),
697
+ })
698
  count += 1
699
  member = r.get("member_name","").strip()
700
  if member:
 
703
  WHERE toLower(p.name) CONTAINS toLower($member)
704
  MATCH (q:ParliamentQuestion {id:$id})
705
  MERGE (p)-[:ASKED_QUESTION]->(q)
706
+ """, {"member": member, "id": node_id})
707
  except Exception as e:
708
  logger.warning(f"[Loader] ParliamentQuestion failed: {e}")
709
  self.stats["parliament_questions"] = count
 
711
  return count
712
 
713
  def load_vigilance_circulars(self, records: list) -> int:
 
714
  count = 0
715
  for r in records:
716
  title = (r.get("title") or "").strip()
 
722
  SET n.title=$title, n.circular_number=$cno,
723
  n.date=$date, n.ministry=$ministry,
724
  n.subject=$subject, n.source=$source,
725
+ n.dataset='cvc', n.scraped_at=$scraped_at
726
+ """, {
727
+ "id": node_id, "title": title,
728
+ "cno": r.get("circular_number",""),
729
+ "date": r.get("date",""),
730
+ "ministry": r.get("ministry",""),
731
+ "subject": r.get("subject",""),
732
+ "source": r.get("source","CVC"),
733
+ "scraped_at":r.get("scraped_at",""),
734
+ })
735
  count += 1
736
  except Exception as e:
737
  logger.warning(f"[Loader] VigilanceCircular failed: {e}")
 
739
  logger.success(f"[Loader] Loaded {count} CVC circulars")
740
  return count
741
 
742
+ # ── BUG-2 FIX: 7 NEW loaders — were scraped but never loaded ─────────────
743
+
744
+ def load_icij_entities(self, records: list) -> int:
745
+ """ICIJ Offshore Leaks entities → ICIJEntity nodes."""
746
+ count = 0
747
+ for r in records:
748
+ name = (r.get("name") or r.get("entity_name") or "").strip()
749
+ if not name: continue
750
+ node_id = make_id(f"icij_{r.get('node_id', name)}")
751
+ try:
752
+ self._run("""
753
+ MERGE (n:ICIJEntity {id:$id})
754
+ SET n.name=$name, n.entity_type=$entity_type,
755
+ n.jurisdiction=$jurisdiction, n.linked_to=$linked_to,
756
+ n.dataset_name=$dataset_name, n.source='ICIJ',
757
+ n.dataset='icij', n.scraped_at=$scraped_at
758
+ """, {
759
+ "id": node_id, "name": name,
760
+ "entity_type": r.get("entity_type", r.get("type","")),
761
+ "jurisdiction": r.get("jurisdiction",""),
762
+ "linked_to": str(r.get("linked_to",[])),
763
+ "dataset_name": r.get("sourceID", r.get("dataset_name","")),
764
+ "scraped_at": r.get("scraped_at",""),
765
+ })
766
+ count += 1
767
+ # Cross-link to Indian politicians/companies by name similarity
768
+ self._run("""
769
+ MATCH (p:Politician)
770
+ WHERE toLower(p.name) CONTAINS toLower($name)
771
+ MATCH (n:ICIJEntity {id:$id})
772
+ MERGE (p)-[:ASSOCIATED_WITH]->(n)
773
+ """, {"name": name, "id": node_id})
774
+ except Exception as e:
775
+ logger.warning(f"[Loader] ICIJEntity failed: {e}")
776
+ self.stats["icij_entities"] = count
777
+ logger.success(f"[Loader] Loaded {count} ICIJ entities")
778
+ return count
779
+
780
+ def load_sanctioned_entities(self, records: list) -> int:
781
+ """OpenSanctions → SanctionedEntity nodes."""
782
+ count = 0
783
+ for r in records:
784
+ name = (r.get("name") or r.get("caption") or "").strip()
785
+ if not name: continue
786
+ node_id = make_id(f"sanction_{r.get('id', name)}")
787
+ try:
788
+ self._run("""
789
+ MERGE (n:SanctionedEntity {id:$id})
790
+ SET n.name=$name, n.schema_type=$schema_type,
791
+ n.topics=$topics, n.countries=$countries,
792
+ n.sanctions_list=$sanctions_list,
793
+ n.source='OpenSanctions', n.dataset='opensanctions',
794
+ n.scraped_at=$scraped_at
795
+ """, {
796
+ "id": node_id, "name": name,
797
+ "schema_type": r.get("schema",""),
798
+ "topics": str(r.get("topics",[])),
799
+ "countries": str(r.get("countries",[])),
800
+ "sanctions_list":r.get("datasets",""),
801
+ "scraped_at": r.get("scraped_at",""),
802
+ })
803
+ count += 1
804
+ self._run("""
805
+ MATCH (p:Politician)
806
+ WHERE toLower(p.name) CONTAINS toLower($name)
807
+ MATCH (n:SanctionedEntity {id:$id})
808
+ MERGE (p)-[:ASSOCIATED_WITH]->(n)
809
+ """, {"name": name, "id": node_id})
810
+ except Exception as e:
811
+ logger.warning(f"[Loader] SanctionedEntity failed: {e}")
812
+ self.stats["sanctioned_entities"] = count
813
+ logger.success(f"[Loader] Loaded {count} sanctioned entities")
814
+ return count
815
+
816
+ def load_court_cases(self, records: list) -> int:
817
+ """NJDG court pendency stats → CourtCase nodes."""
818
+ count = 0
819
+ for r in records:
820
+ court = (r.get("court_name") or r.get("state") or "").strip()
821
+ if not court: continue
822
+ node_id = make_id(f"njdg_{court}_{r.get('year','')}")
823
+ try:
824
+ self._run("""
825
+ MERGE (n:CourtCase {id:$id})
826
+ SET n.court_name=$court, n.state=$state,
827
+ n.pending_cases=$pending, n.disposed_cases=$disposed,
828
+ n.year=$year, n.source='NJDG',
829
+ n.dataset='njdg', n.scraped_at=$scraped_at
830
+ """, {
831
+ "id": node_id, "court": court,
832
+ "state": r.get("state",""),
833
+ "pending": int(r.get("pending_cases",0) or 0),
834
+ "disposed": int(r.get("disposed_cases",0) or 0),
835
+ "year": str(r.get("year","")),
836
+ "scraped_at":r.get("scraped_at",""),
837
+ })
838
+ count += 1
839
+ except Exception as e:
840
+ logger.warning(f"[Loader] CourtCase failed: {e}")
841
+ self.stats["court_cases"] = count
842
+ logger.success(f"[Loader] Loaded {count} court case records")
843
+ return count
844
+
845
+ def load_local_bodies(self, records: list) -> int:
846
+ """LGD (Local Government Directory) → LocalBody nodes."""
847
+ count = 0
848
+ for r in records:
849
+ name = (r.get("name") or r.get("state_name") or "").strip()
850
+ if not name: continue
851
+ node_id = make_id(f"lgd_{r.get('lgd_code',name)}")
852
+ try:
853
+ self._run("""
854
+ MERGE (n:LocalBody {id:$id})
855
+ SET n.name=$name, n.lgd_code=$lgd_code,
856
+ n.entity_type=$entity_type, n.state=$state,
857
+ n.district=$district, n.source='LGD',
858
+ n.dataset='lgd', n.scraped_at=$scraped_at
859
+ """, {
860
+ "id": node_id, "name": name,
861
+ "lgd_code": str(r.get("lgd_code","")),
862
+ "entity_type":r.get("entity_type", r.get("type","")),
863
+ "state": r.get("state",""),
864
+ "district": r.get("district",""),
865
+ "scraped_at": r.get("scraped_at",""),
866
+ })
867
+ count += 1
868
+ except Exception as e:
869
+ logger.warning(f"[Loader] LocalBody failed: {e}")
870
+ self.stats["local_bodies"] = count
871
+ logger.success(f"[Loader] Loaded {count} local bodies")
872
+ return count
873
+
874
+ def load_crime_reports(self, records: list) -> int:
875
+ """NCRB crime statistics — stored as metadata nodes for context."""
876
+ count = 0
877
+ for r in records:
878
+ state = (r.get("state") or "").strip()
879
+ year = str(r.get("year","")).strip()
880
+ if not state or not year: continue
881
+ node_id = make_id(f"ncrb_{state}_{year}")
882
+ try:
883
+ self._run("""
884
+ MERGE (n:CrimeReport {id:$id})
885
+ SET n.state=$state, n.year=$year,
886
+ n.crime_head=$crime_head,
887
+ n.cases_registered=$cases,
888
+ n.source='NCRB', n.dataset='ncrb',
889
+ n.scraped_at=$scraped_at
890
+ """, {
891
+ "id": node_id, "state": state, "year": year,
892
+ "crime_head": r.get("crime_head",""),
893
+ "cases": int(r.get("cases_registered",0) or 0),
894
+ "scraped_at": r.get("scraped_at",""),
895
+ })
896
+ count += 1
897
+ except Exception as e:
898
+ logger.warning(f"[Loader] CrimeReport failed: {e}")
899
+ self.stats["crime_reports"] = count
900
+ logger.success(f"[Loader] Loaded {count} NCRB crime reports")
901
+ return count
902
+
903
+ def load_wikidata_enrichments(self, records: list) -> int:
904
+ """
905
+ Wikidata — enriches EXISTING Politician nodes; does NOT create new ones.
906
+ Uses MATCH not MERGE to avoid phantom nodes.
907
+ """
908
+ count = 0
909
+ for r in records:
910
+ name = (r.get("name") or "").strip()
911
+ if not name: continue
912
+ try:
913
+ result = self._run("""
914
+ MATCH (p:Politician)
915
+ WHERE toLower(p.name) = toLower($name)
916
+ OR any(a IN coalesce(p.aliases,[]) WHERE toLower(a) = toLower($name))
917
+ SET p.wikidata_id = coalesce($wikidata_id, p.wikidata_id),
918
+ p.birth_year = coalesce($birth_year, p.birth_year),
919
+ p.gender = coalesce($gender, p.gender),
920
+ p.positions = coalesce($positions, p.positions),
921
+ p.wikidata_enriched = true
922
+ RETURN count(p) AS updated
923
+ """, {
924
+ "name": name,
925
+ "wikidata_id":r.get("wikidata_id",""),
926
+ "birth_year": str(r.get("birth_year","")),
927
+ "gender": r.get("gender",""),
928
+ "positions": str(r.get("positions",[])),
929
+ })
930
+ if result:
931
+ row = result.single()
932
+ if row and row["updated"] > 0:
933
+ count += 1
934
+ except Exception as e:
935
+ logger.warning(f"[Loader] Wikidata enrichment failed for {name}: {e}")
936
+ self.stats["wikidata_enrichments"] = count
937
+ logger.success(f"[Loader] Wikidata enriched {count} existing Politician nodes")
938
+ return count
939
+
940
+ def load_datagov_documents(self, records: list) -> int:
941
+ """data.gov.in datasets — generic document nodes."""
942
+ count = 0
943
+ for r in records:
944
+ title = (r.get("title") or r.get("resource_title") or "").strip()
945
+ if not title: continue
946
+ node_id = make_id(f"datagov_{r.get('resource_id',title)}")
947
+ try:
948
+ self._run("""
949
+ MERGE (n:DataGovDocument {id:$id})
950
+ SET n.title=$title, n.dataset_name=$dataset_name,
951
+ n.resource_id=$resource_id,
952
+ n.ministry=$ministry,
953
+ n.source='data.gov.in', n.dataset='datagov',
954
+ n.scraped_at=$scraped_at
955
+ """, {
956
+ "id": node_id, "title": title,
957
+ "dataset_name":r.get("_dataset",""),
958
+ "resource_id": r.get("resource_id",""),
959
+ "ministry": r.get("ministry",""),
960
+ "scraped_at": r.get("scraped_at",""),
961
+ })
962
+ count += 1
963
+ except Exception as e:
964
+ logger.warning(f"[Loader] DataGovDocument failed: {e}")
965
+ self.stats["datagov_documents"] = count
966
+ logger.success(f"[Loader] Loaded {count} DataGov documents")
967
+ return count
968
+
969
  def close(self):
970
  if self.driver:
971
  self.driver.close()
 
974
 
975
  if __name__ == "__main__":
976
  parser = argparse.ArgumentParser(description="BharatGraph Graph Loader")
977
+ parser.add_argument("--dry-run", action="store_true")
978
+ parser.add_argument("--file", type=str, default=None)
 
 
979
  args = parser.parse_args()
980
 
981
  print("=" * 55)
 
987
  if args.file:
988
  results = loader.load_from_pipeline_output(args.file)
989
  else:
 
990
  import glob
991
  files = sorted(glob.glob("data/processed/pipeline_*.json"))
992
  if files:
993
+ results = loader.load_from_pipeline_output(files[-1])
 
 
994
  else:
995
+ print("\nNo pipeline output found. Run: python -m processing.pipeline")
 
996
  results = {}
997
 
998
  loader.close()
 
 
 
 
999
  s = loader.stats
1000
+ print(f"\n{'='*55}\nLOAD SUMMARY\n{'='*55}")
1001
+ print(f" Nodes created: {s['nodes_created']}")
1002
+ print(f" Nodes merged: {s['nodes_merged']}")
1003
+ print(f" Relationships: {s['rels_created']}")
1004
+ print(f" Errors: {s['errors']}")
1005
  if args.dry_run:
1006
+ print(" (DRY RUN nothing was written to Neo4j)")
1007
+ print("=" * 55)