abinazebinoy commited on
Commit
6b02e4a
·
1 Parent(s): 7087c8a

fix(B-05 step3, NEW-1): add load_affidavits() to GraphLoader; modern fulltext in setup_schema()

Browse files

B-05: load_affidavits() was completely missing from GraphLoader.
The myneta_scraper scrapes affidavit data but there was no method
to write it to Neo4j as Affidavit nodes with FILED_AFFIDAVIT
relationships. Added full MERGE-based loader with all fields.

NEW-1: setup_schema() updated from deprecated CALL db.index.fulltext.createNodeIndex
to modern CREATE FULLTEXT INDEX IF NOT EXISTS DDL syntax.
Includes Affidavit in the 21-type label list.

Files changed (1) hide show
  1. graph/loader.py +62 -14
graph/loader.py CHANGED
@@ -73,22 +73,22 @@ class GraphLoader:
73
  with self.driver.session() as session:
74
  # BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
75
  try:
 
 
76
  session.run(
77
- "CALL db.index.fulltext.createNodeIndex("
78
- "'globalSearch',"
79
- "['Politician','Company','Contract','AuditReport',"
80
- " 'Scheme','Ministry','Party','PressRelease',"
81
- " 'NGO','ElectoralBond','InsolvencyOrder','Tender',"
82
- " 'RegulatoryOrder','EnforcementAction',"
83
- " 'ParliamentQuestion','VigilanceCircular',"
84
- " 'ICIJEntity','SanctionedEntity','CourtCase','LocalBody'],"
85
- "['name','title','aliases','item_desc','product','buyer_org',"
86
- " 'cin','ministry','summary','seller_name','ngo_name',"
87
- " 'purchaser_name','redeemed_by','company_name','accused',"
88
- " 'entity_name','subject','jurisdiction']"
89
- ")"
90
  )
91
- logger.info("[Loader] Full-text index created or verified (20 types)")
92
  except Exception as e:
93
  logger.debug(f"[Loader] Full-text index note: {e}")
94
 
@@ -404,6 +404,54 @@ class GraphLoader:
404
  logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
405
  return count
406
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
407
  def load_from_pipeline_output(self, filepath: str) -> dict:
408
  """Load everything from a pipeline JSON output file."""
409
  logger.info(f"[Loader] Loading from: {filepath}")
 
73
  with self.driver.session() as session:
74
  # BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
75
  try:
76
+ # NEW-1 FIX: modern CREATE FULLTEXT INDEX (Neo4j 4.3+ / AuraDB)
77
+ # CALL db.index.fulltext.createNodeIndex removed in Neo4j 5.x
78
  session.run(
79
+ "CREATE FULLTEXT INDEX globalSearch IF NOT EXISTS "
80
+ "FOR (n:Politician|Company|Contract|AuditReport|Scheme|Ministry|"
81
+ " Party|PressRelease|NGO|ElectoralBond|InsolvencyOrder|Tender|"
82
+ " RegulatoryOrder|EnforcementAction|ParliamentQuestion|"
83
+ " VigilanceCircular|ICIJEntity|SanctionedEntity|"
84
+ " CourtCase|LocalBody|Affidavit) "
85
+ "ON EACH [n.name, n.title, n.aliases, n.item_desc, n.product, "
86
+ " n.buyer_org, n.cin, n.ministry, n.summary, "
87
+ " n.seller_name, n.ngo_name, n.purchaser_name, "
88
+ " n.redeemed_by, n.company_name, n.accused, "
89
+ " n.entity_name, n.subject, n.jurisdiction]"
 
 
90
  )
91
+ logger.info("[Loader] Full-text index created/verified (21 types, modern syntax)")
92
  except Exception as e:
93
  logger.debug(f"[Loader] Full-text index note: {e}")
94
 
 
404
  logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
405
  return count
406
 
407
+ def load_affidavits(self, records: list) -> int:
408
+ """B-05 FIX: was missing -- affidavit data scraped but never written to graph.
409
+ Investigators query (p:Politician)-[:FILED_AFFIDAVIT]->(a:Affidavit)
410
+ but zero Affidavit nodes existed. All affidavit queries returned empty.
411
+ """
412
+ count = 0
413
+ for r in records:
414
+ pol_id = (r.get("politician_id") or "").strip()
415
+ year = r.get("year")
416
+ if not pol_id or not year:
417
+ continue
418
+ aff_id = make_id(pol_id, str(year))
419
+ result = self._run(
420
+ """
421
+ MERGE (a:Affidavit {id: })
422
+ SET a.politician_id = ,
423
+ a.year = ,
424
+ a.total_assets_crore = ,
425
+ a.movable_assets_crore = ,
426
+ a.liabilities_crore = ,
427
+ a.criminal_cases = ,
428
+ a.source = ,
429
+ a.scraped_at =
430
+ WITH a
431
+ MATCH (p:Politician {id: })
432
+ MERGE (p)-[:FILED_AFFIDAVIT]->(a)
433
+ """,
434
+ {
435
+ "id": aff_id,
436
+ "politician_id": pol_id,
437
+ "year": int(year),
438
+ "total_assets_crore": float(r.get("total_assets_crore") or 0),
439
+ "movable_assets_crore": float(r.get("movable_assets_crore") or 0),
440
+ "liabilities_crore": float(r.get("liabilities_crore") or 0),
441
+ "criminal_cases": int(r.get("criminal_cases") or 0),
442
+ "source": r.get("source", "myneta"),
443
+ "scraped_at": r.get(
444
+ "scraped_at",
445
+ __import__("datetime").datetime.now().isoformat()
446
+ ),
447
+ }
448
+ )
449
+ if result is not None:
450
+ count += 1
451
+ self.stats["nodes_created"] += 1
452
+ logger.info(f"[Loader] Affidavits loaded: {count}")
453
+ return count
454
+
455
  def load_from_pipeline_output(self, filepath: str) -> dict:
456
  """Load everything from a pipeline JSON output file."""
457
  logger.info(f"[Loader] Loading from: {filepath}")