Spaces:
Running
Running
Commit ·
6b02e4a
1
Parent(s): 7087c8a
fix(B-05 step3, NEW-1): add load_affidavits() to GraphLoader; modern fulltext in setup_schema()
Browse filesB-05: load_affidavits() was completely missing from GraphLoader.
The myneta_scraper scrapes affidavit data but there was no method
to write it to Neo4j as Affidavit nodes with FILED_AFFIDAVIT
relationships. Added full MERGE-based loader with all fields.
NEW-1: setup_schema() updated from deprecated CALL db.index.fulltext.createNodeIndex
to modern CREATE FULLTEXT INDEX IF NOT EXISTS DDL syntax.
Includes Affidavit in the 21-type label list.
- graph/loader.py +62 -14
graph/loader.py
CHANGED
|
@@ -73,22 +73,22 @@ class GraphLoader:
|
|
| 73 |
with self.driver.session() as session:
|
| 74 |
# BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
|
| 75 |
try:
|
|
|
|
|
|
|
| 76 |
session.run(
|
| 77 |
-
"
|
| 78 |
-
"
|
| 79 |
-
"
|
| 80 |
-
"
|
| 81 |
-
"
|
| 82 |
-
"
|
| 83 |
-
"
|
| 84 |
-
"
|
| 85 |
-
"
|
| 86 |
-
"
|
| 87 |
-
"
|
| 88 |
-
" 'entity_name','subject','jurisdiction']"
|
| 89 |
-
")"
|
| 90 |
)
|
| 91 |
-
logger.info("[Loader] Full-text index created
|
| 92 |
except Exception as e:
|
| 93 |
logger.debug(f"[Loader] Full-text index note: {e}")
|
| 94 |
|
|
@@ -404,6 +404,54 @@ class GraphLoader:
|
|
| 404 |
logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
|
| 405 |
return count
|
| 406 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
def load_from_pipeline_output(self, filepath: str) -> dict:
|
| 408 |
"""Load everything from a pipeline JSON output file."""
|
| 409 |
logger.info(f"[Loader] Loading from: {filepath}")
|
|
|
|
| 73 |
with self.driver.session() as session:
|
| 74 |
# BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
|
| 75 |
try:
|
| 76 |
+
# NEW-1 FIX: modern CREATE FULLTEXT INDEX (Neo4j 4.3+ / AuraDB)
|
| 77 |
+
# CALL db.index.fulltext.createNodeIndex removed in Neo4j 5.x
|
| 78 |
session.run(
|
| 79 |
+
"CREATE FULLTEXT INDEX globalSearch IF NOT EXISTS "
|
| 80 |
+
"FOR (n:Politician|Company|Contract|AuditReport|Scheme|Ministry|"
|
| 81 |
+
" Party|PressRelease|NGO|ElectoralBond|InsolvencyOrder|Tender|"
|
| 82 |
+
" RegulatoryOrder|EnforcementAction|ParliamentQuestion|"
|
| 83 |
+
" VigilanceCircular|ICIJEntity|SanctionedEntity|"
|
| 84 |
+
" CourtCase|LocalBody|Affidavit) "
|
| 85 |
+
"ON EACH [n.name, n.title, n.aliases, n.item_desc, n.product, "
|
| 86 |
+
" n.buyer_org, n.cin, n.ministry, n.summary, "
|
| 87 |
+
" n.seller_name, n.ngo_name, n.purchaser_name, "
|
| 88 |
+
" n.redeemed_by, n.company_name, n.accused, "
|
| 89 |
+
" n.entity_name, n.subject, n.jurisdiction]"
|
|
|
|
|
|
|
| 90 |
)
|
| 91 |
+
logger.info("[Loader] Full-text index created/verified (21 types, modern syntax)")
|
| 92 |
except Exception as e:
|
| 93 |
logger.debug(f"[Loader] Full-text index note: {e}")
|
| 94 |
|
|
|
|
| 404 |
logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
|
| 405 |
return count
|
| 406 |
|
| 407 |
+
def load_affidavits(self, records: list) -> int:
|
| 408 |
+
"""B-05 FIX: was missing -- affidavit data scraped but never written to graph.
|
| 409 |
+
Investigators query (p:Politician)-[:FILED_AFFIDAVIT]->(a:Affidavit)
|
| 410 |
+
but zero Affidavit nodes existed. All affidavit queries returned empty.
|
| 411 |
+
"""
|
| 412 |
+
count = 0
|
| 413 |
+
for r in records:
|
| 414 |
+
pol_id = (r.get("politician_id") or "").strip()
|
| 415 |
+
year = r.get("year")
|
| 416 |
+
if not pol_id or not year:
|
| 417 |
+
continue
|
| 418 |
+
aff_id = make_id(pol_id, str(year))
|
| 419 |
+
result = self._run(
|
| 420 |
+
"""
|
| 421 |
+
MERGE (a:Affidavit {id: })
|
| 422 |
+
SET a.politician_id = ,
|
| 423 |
+
a.year = ,
|
| 424 |
+
a.total_assets_crore = ,
|
| 425 |
+
a.movable_assets_crore = ,
|
| 426 |
+
a.liabilities_crore = ,
|
| 427 |
+
a.criminal_cases = ,
|
| 428 |
+
a.source = ,
|
| 429 |
+
a.scraped_at =
|
| 430 |
+
WITH a
|
| 431 |
+
MATCH (p:Politician {id: })
|
| 432 |
+
MERGE (p)-[:FILED_AFFIDAVIT]->(a)
|
| 433 |
+
""",
|
| 434 |
+
{
|
| 435 |
+
"id": aff_id,
|
| 436 |
+
"politician_id": pol_id,
|
| 437 |
+
"year": int(year),
|
| 438 |
+
"total_assets_crore": float(r.get("total_assets_crore") or 0),
|
| 439 |
+
"movable_assets_crore": float(r.get("movable_assets_crore") or 0),
|
| 440 |
+
"liabilities_crore": float(r.get("liabilities_crore") or 0),
|
| 441 |
+
"criminal_cases": int(r.get("criminal_cases") or 0),
|
| 442 |
+
"source": r.get("source", "myneta"),
|
| 443 |
+
"scraped_at": r.get(
|
| 444 |
+
"scraped_at",
|
| 445 |
+
__import__("datetime").datetime.now().isoformat()
|
| 446 |
+
),
|
| 447 |
+
}
|
| 448 |
+
)
|
| 449 |
+
if result is not None:
|
| 450 |
+
count += 1
|
| 451 |
+
self.stats["nodes_created"] += 1
|
| 452 |
+
logger.info(f"[Loader] Affidavits loaded: {count}")
|
| 453 |
+
return count
|
| 454 |
+
|
| 455 |
def load_from_pipeline_output(self, filepath: str) -> dict:
|
| 456 |
"""Load everything from a pipeline JSON output file."""
|
| 457 |
logger.info(f"[Loader] Loading from: {filepath}")
|