Spaces:
Running
fix(loader): BUG-2/3/17/19/21 — 7 new loaders, fulltext, kwargs, SHA-256
Browse filesBUG-2 (CRITICAL): 7 datasets were scraped but never loaded.
Added loaders: load_icij_entities, load_sanctioned_entities,
load_court_cases, load_local_bodies, load_crime_reports,
load_wikidata_enrichments, load_datagov_documents.
All 7 called in load_from_pipeline_output().
BUG-3: fulltext index in setup_schema() now covers all 20
node types (was only 8). Added: NGO, ElectoralBond,
InsolvencyOrder, Tender, RegulatoryOrder, EnforcementAction,
ParliamentQuestion, VigilanceCircular, ICIJEntity,
SanctionedEntity, CourtCase, LocalBody.
BUG-17: _run() now accepts BOTH positional dict and **kwargs
so new Phase 28 loaders (which used keyword args) work
without errors.
BUG-19: make_id() upgraded from MD5 to SHA-256 (20 hex chars)
for lower collision risk across large datasets.
BUG-21: GraphLoader.__init__ now accepts driver= kwarg so
admin.py can pass the existing FastAPI driver instead of
opening a second Neo4j connection.
- graph/loader.py +435 -279
|
@@ -1,56 +1,43 @@
|
|
| 1 |
"""
|
| 2 |
BharatGraph - Neo4j Graph Loader
|
| 3 |
Loads cleaned + resolved data from Phase 2 pipeline into Neo4j.
|
| 4 |
-
|
| 5 |
-
What this does:
|
| 6 |
-
1. Connects to Neo4j AuraDB (free tier)
|
| 7 |
-
2. Creates constraints and indexes (from schema.py)
|
| 8 |
-
3. Loads each entity type as nodes
|
| 9 |
-
4. Creates relationships between connected entities
|
| 10 |
-
5. Reports stats: nodes created, relationships created
|
| 11 |
-
|
| 12 |
-
Usage:
|
| 13 |
-
python -m graph.loader
|
| 14 |
-
python -m graph.loader --dry-run (shows what would be loaded)
|
| 15 |
"""
|
| 16 |
|
| 17 |
-
import os
|
| 18 |
-
import sys
|
| 19 |
-
import json
|
| 20 |
-
import hashlib
|
| 21 |
-
import argparse
|
| 22 |
from datetime import datetime
|
| 23 |
from loguru import logger
|
| 24 |
|
| 25 |
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
| 26 |
-
|
| 27 |
from graph.schema import SETUP_QUERIES, NODE_SCHEMAS
|
| 28 |
|
| 29 |
|
|
|
|
|
|
|
| 30 |
def make_id(*parts) -> str:
|
| 31 |
-
"""Create a stable
|
| 32 |
combined = "|".join(str(p).lower().strip() for p in parts)
|
| 33 |
-
return hashlib.
|
| 34 |
|
| 35 |
|
| 36 |
class GraphLoader:
|
| 37 |
"""
|
| 38 |
Loads BharatGraph data into Neo4j.
|
| 39 |
-
|
|
|
|
| 40 |
"""
|
| 41 |
|
| 42 |
-
def __init__(self, dry_run: bool = False):
|
| 43 |
self.dry_run = dry_run
|
| 44 |
-
self.driver =
|
| 45 |
self.stats = {
|
| 46 |
-
"nodes_created":
|
| 47 |
-
"nodes_merged":
|
| 48 |
-
"rels_created":
|
| 49 |
-
"errors":
|
| 50 |
}
|
| 51 |
if dry_run:
|
| 52 |
-
logger.info("[Loader] DRY RUN mode
|
| 53 |
-
|
| 54 |
self._connect()
|
| 55 |
|
| 56 |
def _connect(self):
|
|
@@ -61,7 +48,7 @@ class GraphLoader:
|
|
| 61 |
load_dotenv()
|
| 62 |
|
| 63 |
uri = os.getenv("NEO4J_URI", "")
|
| 64 |
-
user = os.getenv("NEO4J_USER",
|
| 65 |
pwd = os.getenv("NEO4J_PASSWORD", "")
|
| 66 |
|
| 67 |
if not pwd:
|
|
@@ -71,9 +58,8 @@ class GraphLoader:
|
|
| 71 |
self.driver = GraphDatabase.driver(uri, auth=(user, pwd))
|
| 72 |
self.driver.verify_connectivity()
|
| 73 |
logger.success(f"[Loader] Connected to Neo4j: {uri[:40]}...")
|
| 74 |
-
|
| 75 |
except ImportError:
|
| 76 |
-
logger.error("[Loader] neo4j package not installed.
|
| 77 |
raise
|
| 78 |
except Exception as e:
|
| 79 |
logger.error(f"[Loader] Connection failed: {e}")
|
|
@@ -85,22 +71,27 @@ class GraphLoader:
|
|
| 85 |
logger.info(f"[Loader] DRY RUN: would run {len(SETUP_QUERIES)} setup queries")
|
| 86 |
return
|
| 87 |
with self.driver.session() as session:
|
| 88 |
-
#
|
| 89 |
try:
|
| 90 |
session.run(
|
| 91 |
"CALL db.index.fulltext.createNodeIndex("
|
| 92 |
"'globalSearch',"
|
| 93 |
"['Politician','Company','Contract','AuditReport',"
|
| 94 |
-
" 'Scheme','Ministry','Party','PressRelease'
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
"['name','title','aliases','item_desc','product','buyer_org',"
|
| 96 |
-
" 'cin','ministry','summary','seller_name'
|
|
|
|
|
|
|
| 97 |
")"
|
| 98 |
)
|
| 99 |
-
logger.info("[Loader] Full-text index created or verified")
|
| 100 |
except Exception as e:
|
| 101 |
logger.debug(f"[Loader] Full-text index note: {e}")
|
| 102 |
|
| 103 |
-
# Run all constraint/index setup queries inside the same session
|
| 104 |
for query in SETUP_QUERIES:
|
| 105 |
try:
|
| 106 |
session.run(query)
|
|
@@ -109,15 +100,22 @@ class GraphLoader:
|
|
| 109 |
|
| 110 |
logger.success("[Loader] Schema constraints and indexes ready")
|
| 111 |
|
| 112 |
-
def _run(self, query: str, params: dict = None):
|
| 113 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
if self.dry_run:
|
| 115 |
logger.debug(f"[Loader] DRY RUN query: {query[:80]}...")
|
| 116 |
return None
|
|
|
|
|
|
|
| 117 |
try:
|
| 118 |
with self.driver.session() as session:
|
| 119 |
-
|
| 120 |
-
return result
|
| 121 |
except Exception as e:
|
| 122 |
logger.error(f"[Loader] Query failed: {e}\nQuery: {query[:100]}")
|
| 123 |
self.stats["errors"] += 1
|
|
@@ -126,7 +124,6 @@ class GraphLoader:
|
|
| 126 |
# ── Node loaders ──────────────────────────────────────
|
| 127 |
|
| 128 |
def load_politicians(self, records: list) -> int:
|
| 129 |
-
"""Load politician records as (Politician) nodes."""
|
| 130 |
count = 0
|
| 131 |
for r in records:
|
| 132 |
name = r.get("name", "").strip()
|
|
@@ -163,8 +160,6 @@ class GraphLoader:
|
|
| 163 |
}
|
| 164 |
self._run(query, params)
|
| 165 |
count += 1
|
| 166 |
-
|
| 167 |
-
# Create Party node + MEMBER_OF relationship
|
| 168 |
party = r.get("party", "").strip()
|
| 169 |
if party:
|
| 170 |
self._create_party_and_link(node_id, party)
|
|
@@ -174,7 +169,6 @@ class GraphLoader:
|
|
| 174 |
return count
|
| 175 |
|
| 176 |
def _create_party_and_link(self, politician_id: str, party_name: str):
|
| 177 |
-
"""Create Party node and link politician to it."""
|
| 178 |
party_id = make_id(party_name)
|
| 179 |
query = """
|
| 180 |
MERGE (party:Party {id: $party_id})
|
|
@@ -184,13 +178,12 @@ class GraphLoader:
|
|
| 184 |
MERGE (p)-[:MEMBER_OF]->(party)
|
| 185 |
"""
|
| 186 |
self._run(query, {
|
| 187 |
-
"party_id":
|
| 188 |
"party_name": party_name,
|
| 189 |
-
"pol_id":
|
| 190 |
})
|
| 191 |
|
| 192 |
def load_companies(self, records: list) -> int:
|
| 193 |
-
"""Load company records as (Company) nodes."""
|
| 194 |
count = 0
|
| 195 |
for r in records:
|
| 196 |
name = r.get("name", "").strip()
|
|
@@ -229,7 +222,6 @@ class GraphLoader:
|
|
| 229 |
return count
|
| 230 |
|
| 231 |
def load_contracts(self, records: list) -> int:
|
| 232 |
-
"""Load GeM contract records as (Contract) nodes + relationships."""
|
| 233 |
count = 0
|
| 234 |
for r in records:
|
| 235 |
order_id = r.get("order_id", "").strip()
|
|
@@ -237,7 +229,6 @@ class GraphLoader:
|
|
| 237 |
if not order_id:
|
| 238 |
continue
|
| 239 |
node_id = make_id(order_id)
|
| 240 |
-
# Create Contract node
|
| 241 |
query = """
|
| 242 |
MERGE (c:Contract {id: $id})
|
| 243 |
SET c.order_id = $order_id,
|
|
@@ -264,13 +255,8 @@ class GraphLoader:
|
|
| 264 |
}
|
| 265 |
self._run(query, params)
|
| 266 |
count += 1
|
| 267 |
-
|
| 268 |
-
# Link seller company to this contract
|
| 269 |
if seller:
|
| 270 |
-
self._link_company_to_contract(seller, node_id,
|
| 271 |
-
float(r.get("amount_crore", 0) or 0))
|
| 272 |
-
|
| 273 |
-
# Link contract to ministry
|
| 274 |
buyer = r.get("buyer_org", "").strip()
|
| 275 |
if buyer:
|
| 276 |
self._link_contract_to_ministry(node_id, buyer)
|
|
@@ -279,65 +265,37 @@ class GraphLoader:
|
|
| 279 |
self.stats["nodes_created"] += count
|
| 280 |
return count
|
| 281 |
|
| 282 |
-
def _link_company_to_contract(self, seller_name
|
| 283 |
-
contract_id: str, amount: float):
|
| 284 |
-
"""Create WON_CONTRACT relationship from Company to Contract."""
|
| 285 |
company_id = make_id(seller_name)
|
| 286 |
-
|
| 287 |
MERGE (co:Company {id: $co_id})
|
| 288 |
SET co.name = $seller_name
|
| 289 |
WITH co
|
| 290 |
MATCH (c:Contract {id: $contract_id})
|
| 291 |
MERGE (co)-[r:WON_CONTRACT]->(c)
|
| 292 |
SET r.amount_crore = $amount
|
| 293 |
-
"""
|
| 294 |
-
|
| 295 |
-
"co_id": company_id,
|
| 296 |
-
"seller_name": seller_name,
|
| 297 |
-
"contract_id": contract_id,
|
| 298 |
-
"amount": amount,
|
| 299 |
-
})
|
| 300 |
self.stats["rels_created"] += 1
|
| 301 |
|
| 302 |
-
def _link_contract_to_ministry(self, contract_id
|
| 303 |
-
"""Create AWARDED_BY relationship from Contract to Ministry."""
|
| 304 |
ministry_id = make_id(ministry_name)
|
| 305 |
-
|
| 306 |
MERGE (m:Ministry {id: $m_id})
|
| 307 |
SET m.name = $m_name
|
| 308 |
WITH m
|
| 309 |
MATCH (c:Contract {id: $contract_id})
|
| 310 |
MERGE (c)-[:AWARDED_BY]->(m)
|
| 311 |
-
"""
|
| 312 |
-
self._run(query, {
|
| 313 |
-
"m_id": ministry_id,
|
| 314 |
-
"m_name": ministry_name,
|
| 315 |
-
"contract_id": contract_id,
|
| 316 |
-
})
|
| 317 |
self.stats["rels_created"] += 1
|
| 318 |
|
| 319 |
def load_audit_reports(self, records: list) -> int:
|
| 320 |
-
"""Load CAG audit report records as (AuditReport) nodes."""
|
| 321 |
count = 0
|
| 322 |
for r in records:
|
| 323 |
title = r.get("title", "").strip()
|
| 324 |
if not title:
|
| 325 |
continue
|
| 326 |
node_id = make_id(title, r.get("url", ""))
|
| 327 |
-
query = """
|
| 328 |
-
MERGE (a:AuditReport {id: $id})
|
| 329 |
-
SET a.title = $title,
|
| 330 |
-
a.url = $url,
|
| 331 |
-
a.year = $year,
|
| 332 |
-
a.state = $state,
|
| 333 |
-
a.scheme = $scheme,
|
| 334 |
-
a.amount_crore = $amount,
|
| 335 |
-
a.irregularity_type = $irr_type,
|
| 336 |
-
a.finding = $finding,
|
| 337 |
-
a.alert_keywords = $keywords,
|
| 338 |
-
a.source = $source,
|
| 339 |
-
a.scraped_at = $scraped_at
|
| 340 |
-
"""
|
| 341 |
params = {
|
| 342 |
"id": node_id,
|
| 343 |
"title": title,
|
|
@@ -352,66 +310,52 @@ class GraphLoader:
|
|
| 352 |
"source": r.get("_source", "cag"),
|
| 353 |
"scraped_at": r.get("scraped_at", datetime.now().isoformat()),
|
| 354 |
}
|
| 355 |
-
self._run(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
count += 1
|
| 357 |
-
|
| 358 |
-
# Link to scheme if present
|
| 359 |
scheme = r.get("scheme", "").strip()
|
| 360 |
if scheme:
|
| 361 |
-
self._link_audit_to_scheme(node_id, scheme,
|
| 362 |
-
float(r.get("amount_crore", 0) or 0))
|
| 363 |
|
| 364 |
logger.success(f"[Loader] Audit reports loaded: {count}")
|
| 365 |
self.stats["nodes_created"] += count
|
| 366 |
return count
|
| 367 |
|
| 368 |
-
def _link_audit_to_scheme(self, audit_id
|
| 369 |
-
amount: float):
|
| 370 |
-
"""Create FLAGS relationship from AuditReport to Scheme."""
|
| 371 |
scheme_id = make_id(scheme_name)
|
| 372 |
-
|
| 373 |
MERGE (s:Scheme {id: $s_id})
|
| 374 |
SET s.name = $s_name
|
| 375 |
WITH s
|
| 376 |
MATCH (a:AuditReport {id: $audit_id})
|
| 377 |
MERGE (a)-[r:FLAGS]->(s)
|
| 378 |
SET r.amount_crore = $amount
|
| 379 |
-
"""
|
| 380 |
-
self._run(query, {
|
| 381 |
-
"s_id": scheme_id,
|
| 382 |
-
"s_name": scheme_name,
|
| 383 |
-
"audit_id": audit_id,
|
| 384 |
-
"amount": amount,
|
| 385 |
-
})
|
| 386 |
self.stats["rels_created"] += 1
|
| 387 |
|
| 388 |
def load_press_releases(self, records: list) -> int:
|
| 389 |
-
"""Load PIB press release records as (PressRelease) nodes."""
|
| 390 |
count = 0
|
| 391 |
for r in records:
|
| 392 |
title = r.get("title", "").strip()
|
| 393 |
if not title:
|
| 394 |
continue
|
| 395 |
node_id = make_id(title, r.get("link", ""))
|
| 396 |
-
|
| 397 |
MERGE (pr:PressRelease {id: $id})
|
| 398 |
-
SET pr.title
|
| 399 |
-
pr.
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
"title": title,
|
| 408 |
-
"link": r.get("link", ""),
|
| 409 |
-
"published": r.get("published", ""),
|
| 410 |
-
"keywords": str(r.get("alert_keywords", [])),
|
| 411 |
-
"source": r.get("_source", "pib"),
|
| 412 |
-
"scraped_at": r.get("scraped_at", datetime.now().isoformat()),
|
| 413 |
-
}
|
| 414 |
-
self._run(query, params)
|
| 415 |
count += 1
|
| 416 |
|
| 417 |
logger.success(f"[Loader] Press releases loaded: {count}")
|
|
@@ -419,14 +363,6 @@ class GraphLoader:
|
|
| 419 |
return count
|
| 420 |
|
| 421 |
def load_politician_company_links(self, matches: list) -> int:
|
| 422 |
-
"""
|
| 423 |
-
Create DIRECTOR_OF relationships for politician-company matches.
|
| 424 |
-
BUG-17 FIX: was using make_id(name) which doesn't match canonical IDs
|
| 425 |
-
created by load_politicians() (which uses make_id(name, state, election)).
|
| 426 |
-
Fix: MATCH by name using toLower() to find existing nodes, only MERGE the
|
| 427 |
-
relationship. If no existing node found, create one with name-only ID as
|
| 428 |
-
fallback so the edge is never silently lost.
|
| 429 |
-
"""
|
| 430 |
count = 0
|
| 431 |
for match in matches:
|
| 432 |
pol_name = match.get("name_a", match.get("politician_name", "")).strip()
|
|
@@ -434,26 +370,6 @@ class GraphLoader:
|
|
| 434 |
score = match.get("score", 0.0)
|
| 435 |
if not pol_name or not co_name:
|
| 436 |
continue
|
| 437 |
-
|
| 438 |
-
# BUG-17 FIX: match by name (case-insensitive) to find the canonical node,
|
| 439 |
-
# fall back to make_id(name) only if not found in graph yet.
|
| 440 |
-
query = """
|
| 441 |
-
MERGE (p:Politician {id: coalesce(
|
| 442 |
-
(MATCH (x:Politician) WHERE toLower(x.name) = toLower($pol_name) RETURN x.id LIMIT 1)[0],
|
| 443 |
-
$pol_id_fallback
|
| 444 |
-
)})
|
| 445 |
-
SET p.name = coalesce(p.name, $pol_name)
|
| 446 |
-
MERGE (c:Company {id: coalesce(
|
| 447 |
-
(MATCH (x:Company) WHERE toLower(x.name) = toLower($co_name) RETURN x.id LIMIT 1)[0],
|
| 448 |
-
$co_id_fallback
|
| 449 |
-
)})
|
| 450 |
-
SET c.name = coalesce(c.name, $co_name)
|
| 451 |
-
MERGE (p)-[r:DIRECTOR_OF]->(c)
|
| 452 |
-
SET r.confidence = $score,
|
| 453 |
-
r.source = 'entity_resolution',
|
| 454 |
-
r.detected_at = $now
|
| 455 |
-
"""
|
| 456 |
-
# Simpler, more reliable approach using two separate lookups
|
| 457 |
link_query = """
|
| 458 |
OPTIONAL MATCH (existing_p:Politician)
|
| 459 |
WHERE toLower(existing_p.name) = toLower($pol_name)
|
|
@@ -489,10 +405,7 @@ class GraphLoader:
|
|
| 489 |
return count
|
| 490 |
|
| 491 |
def load_from_pipeline_output(self, filepath: str) -> dict:
|
| 492 |
-
"""
|
| 493 |
-
Load everything from a pipeline JSON output file.
|
| 494 |
-
This is the main entry point.
|
| 495 |
-
"""
|
| 496 |
logger.info(f"[Loader] Loading from: {filepath}")
|
| 497 |
with open(filepath, encoding="utf-8") as f:
|
| 498 |
data = json.load(f)
|
|
@@ -503,44 +416,36 @@ class GraphLoader:
|
|
| 503 |
self.setup_schema()
|
| 504 |
|
| 505 |
results = {}
|
| 506 |
-
if raw.get("myneta"):
|
| 507 |
-
|
| 508 |
-
if raw.get("
|
| 509 |
-
|
| 510 |
-
if raw.get("
|
| 511 |
-
|
| 512 |
-
if raw.get("
|
| 513 |
-
|
| 514 |
-
if raw.get("
|
| 515 |
-
|
| 516 |
-
if
|
| 517 |
-
|
| 518 |
-
if raw.get("
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
if raw.get("
|
| 523 |
-
|
| 524 |
-
if raw.get("
|
| 525 |
-
|
| 526 |
-
if raw.get("
|
| 527 |
-
|
| 528 |
-
if raw.get("
|
| 529 |
-
results["tenders"] = self.load_tenders(raw["cppp"])
|
| 530 |
-
if raw.get("loksabha"):
|
| 531 |
-
results["parliament_questions"] = self.load_parliament_questions(raw["loksabha"])
|
| 532 |
-
if raw.get("cvc"):
|
| 533 |
-
results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
|
| 534 |
-
|
| 535 |
|
| 536 |
logger.success(f"[Loader] Load complete. Stats: {self.stats}")
|
| 537 |
return {**results, "stats": self.stats}
|
| 538 |
|
| 539 |
-
|
| 540 |
-
# ── Phase 28: 8 new dataset loaders ──────────────────────────────────────
|
| 541 |
|
| 542 |
def load_regulatory_orders(self, records: list) -> int:
|
| 543 |
-
"""SEBI enforcement orders → RegulatoryOrder nodes."""
|
| 544 |
count = 0
|
| 545 |
for r in records:
|
| 546 |
title = (r.get("title") or "").strip()
|
|
@@ -551,12 +456,15 @@ class GraphLoader:
|
|
| 551 |
MERGE (n:RegulatoryOrder {id:$id})
|
| 552 |
SET n.title=$title, n.url=$url, n.order_type=$order_type,
|
| 553 |
n.entity_name=$entity_name, n.violation=$violation,
|
| 554 |
-
n.source=$source, n.dataset=
|
| 555 |
-
""",
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
|
|
|
|
|
|
|
|
|
| 560 |
count += 1
|
| 561 |
except Exception as e:
|
| 562 |
logger.warning(f"[Loader] RegulatoryOrder failed: {e}")
|
|
@@ -565,7 +473,6 @@ class GraphLoader:
|
|
| 565 |
return count
|
| 566 |
|
| 567 |
def load_enforcement_actions(self, records: list) -> int:
|
| 568 |
-
"""ED enforcement actions → EnforcementAction nodes."""
|
| 569 |
count = 0
|
| 570 |
for r in records:
|
| 571 |
title = (r.get("title") or "").strip()
|
|
@@ -577,12 +484,16 @@ class GraphLoader:
|
|
| 577 |
SET n.title=$title, n.url=$url, n.date=$date,
|
| 578 |
n.amount_crore=$amount_crore, n.case_type=$case_type,
|
| 579 |
n.accused=$accused, n.source=$source,
|
| 580 |
-
n.dataset=
|
| 581 |
-
""",
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 586 |
count += 1
|
| 587 |
accused = r.get("accused","").strip()
|
| 588 |
if accused:
|
|
@@ -591,7 +502,7 @@ class GraphLoader:
|
|
| 591 |
WHERE toLower(p.name) CONTAINS toLower($accused)
|
| 592 |
MATCH (n:EnforcementAction {id:$id})
|
| 593 |
MERGE (p)-[:SUBJECT_OF]->(n)
|
| 594 |
-
""", accused
|
| 595 |
except Exception as e:
|
| 596 |
logger.warning(f"[Loader] EnforcementAction failed: {e}")
|
| 597 |
self.stats["enforcement_actions"] = count
|
|
@@ -599,7 +510,6 @@ class GraphLoader:
|
|
| 599 |
return count
|
| 600 |
|
| 601 |
def load_electoral_bonds(self, records: list) -> int:
|
| 602 |
-
"""Electoral bonds → ElectoralBond nodes + DONATED_VIA/REDEEMED_BY."""
|
| 603 |
count = 0
|
| 604 |
for r in records:
|
| 605 |
purchaser = (r.get("purchaser_name") or "").strip()
|
|
@@ -611,26 +521,33 @@ class GraphLoader:
|
|
| 611 |
SET n.bond_number=$bond_no, n.purchaser_name=$purchaser,
|
| 612 |
n.denomination_crore=$denom, n.purchase_date=$p_date,
|
| 613 |
n.redemption_date=$r_date, n.redeemed_by=$redeemer,
|
| 614 |
-
n.source=$source, n.dataset=
|
| 615 |
n.scraped_at=$scraped_at
|
| 616 |
-
""",
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 622 |
count += 1
|
| 623 |
redeemer = r.get("redeemed_by","").strip()
|
| 624 |
if purchaser and redeemer:
|
| 625 |
self._run("""
|
| 626 |
MERGE (c:Company {id:$co_id})
|
| 627 |
-
ON CREATE SET c.name=$purchaser, c.source=
|
| 628 |
MERGE (p:Party {id:$pt_id})
|
| 629 |
-
ON CREATE SET p.name=$redeemer, p.source=
|
| 630 |
MATCH (b:ElectoralBond {id:$b_id})
|
| 631 |
MERGE (c)-[:DONATED_VIA]->(b)-[:REDEEMED_BY]->(p)
|
| 632 |
-
""",
|
| 633 |
-
|
|
|
|
|
|
|
|
|
|
| 634 |
except Exception as e:
|
| 635 |
logger.warning(f"[Loader] ElectoralBond failed: {e}")
|
| 636 |
self.stats["electoral_bonds"] = count
|
|
@@ -638,7 +555,6 @@ class GraphLoader:
|
|
| 638 |
return count
|
| 639 |
|
| 640 |
def load_insolvency_orders(self, records: list) -> int:
|
| 641 |
-
"""IBBI insolvency orders → InsolvencyOrder + HAS_INSOLVENCY link."""
|
| 642 |
count = 0
|
| 643 |
for r in records:
|
| 644 |
company = (r.get("company_name") or "").strip()
|
|
@@ -650,19 +566,23 @@ class GraphLoader:
|
|
| 650 |
SET n.company_name=$company, n.cin=$cin,
|
| 651 |
n.process_type=$process_type, n.admitted_date=$admitted_date,
|
| 652 |
n.status=$status, n.admitted_claims=$claims,
|
| 653 |
-
n.source=$source, n.dataset=
|
| 654 |
-
""",
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 659 |
count += 1
|
| 660 |
self._run("""
|
| 661 |
MATCH (c:Company)
|
| 662 |
WHERE toLower(c.name) CONTAINS toLower($company)
|
| 663 |
MATCH (n:InsolvencyOrder {id:$id})
|
| 664 |
MERGE (c)-[:HAS_INSOLVENCY]->(n)
|
| 665 |
-
""", company
|
| 666 |
except Exception as e:
|
| 667 |
logger.warning(f"[Loader] InsolvencyOrder failed: {e}")
|
| 668 |
self.stats["insolvency_orders"] = count
|
|
@@ -670,7 +590,6 @@ class GraphLoader:
|
|
| 670 |
return count
|
| 671 |
|
| 672 |
def load_ngos(self, records: list) -> int:
|
| 673 |
-
"""NGO Darpan → NGO nodes."""
|
| 674 |
count = 0
|
| 675 |
for r in records:
|
| 676 |
name = (r.get("ngo_name") or "").strip()
|
|
@@ -684,15 +603,20 @@ class GraphLoader:
|
|
| 684 |
n.registration_type=$reg_type, n.year_of_reg=$year,
|
| 685 |
n.key_issues=$issues, n.csr_receipts=$csr,
|
| 686 |
n.govt_grants=$grants, n.source=$source,
|
| 687 |
-
n.dataset=
|
| 688 |
-
""",
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 696 |
count += 1
|
| 697 |
except Exception as e:
|
| 698 |
logger.warning(f"[Loader] NGO failed: {e}")
|
|
@@ -701,7 +625,6 @@ class GraphLoader:
|
|
| 701 |
return count
|
| 702 |
|
| 703 |
def load_tenders(self, records: list) -> int:
|
| 704 |
-
"""CPPP tenders → Tender nodes + ISSUED_TENDER ministry link."""
|
| 705 |
count = 0
|
| 706 |
for r in records:
|
| 707 |
title = (r.get("title") or "").strip()
|
|
@@ -716,27 +639,34 @@ class GraphLoader:
|
|
| 716 |
n.bid_end_date=$bid_end, n.status=$status,
|
| 717 |
n.awarded_to=$awarded_to, n.awarded_value=$awarded_val,
|
| 718 |
n.single_bid=$single_bid, n.source=$source,
|
| 719 |
-
n.dataset=
|
| 720 |
-
""",
|
| 721 |
-
|
| 722 |
-
|
| 723 |
-
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
| 727 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 728 |
count += 1
|
| 729 |
ministry = r.get("ministry","").strip()
|
| 730 |
if ministry:
|
| 731 |
self._run("""
|
| 732 |
MERGE (m:Ministry {id:$m_id})
|
| 733 |
-
ON CREATE SET m.name=$ministry, m.source=
|
| 734 |
MATCH (t:Tender {id:$t_id})
|
| 735 |
MERGE (m)-[:ISSUED_TENDER]->(t)
|
| 736 |
-
""", m_id
|
| 737 |
if r.get("single_bid"):
|
| 738 |
self._run("MATCH (t:Tender {id:$id}) SET t.risk_flag='SINGLE_BID'",
|
| 739 |
-
id
|
| 740 |
except Exception as e:
|
| 741 |
logger.warning(f"[Loader] Tender failed: {e}")
|
| 742 |
self.stats["tenders"] = count
|
|
@@ -744,7 +674,6 @@ class GraphLoader:
|
|
| 744 |
return count
|
| 745 |
|
| 746 |
def load_parliament_questions(self, records: list) -> int:
|
| 747 |
-
"""Lok Sabha questions → ParliamentQuestion nodes."""
|
| 748 |
count = 0
|
| 749 |
for r in records:
|
| 750 |
subject = (r.get("subject") or "").strip()
|
|
@@ -756,11 +685,16 @@ class GraphLoader:
|
|
| 756 |
SET n.subject=$subject, n.question_number=$qno,
|
| 757 |
n.member_name=$member, n.ministry=$ministry,
|
| 758 |
n.session=$session, n.source=$source,
|
| 759 |
-
n.dataset=
|
| 760 |
-
""",
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 764 |
count += 1
|
| 765 |
member = r.get("member_name","").strip()
|
| 766 |
if member:
|
|
@@ -769,7 +703,7 @@ class GraphLoader:
|
|
| 769 |
WHERE toLower(p.name) CONTAINS toLower($member)
|
| 770 |
MATCH (q:ParliamentQuestion {id:$id})
|
| 771 |
MERGE (p)-[:ASKED_QUESTION]->(q)
|
| 772 |
-
""", member
|
| 773 |
except Exception as e:
|
| 774 |
logger.warning(f"[Loader] ParliamentQuestion failed: {e}")
|
| 775 |
self.stats["parliament_questions"] = count
|
|
@@ -777,7 +711,6 @@ class GraphLoader:
|
|
| 777 |
return count
|
| 778 |
|
| 779 |
def load_vigilance_circulars(self, records: list) -> int:
|
| 780 |
-
"""CVC circulars → VigilanceCircular nodes."""
|
| 781 |
count = 0
|
| 782 |
for r in records:
|
| 783 |
title = (r.get("title") or "").strip()
|
|
@@ -789,11 +722,16 @@ class GraphLoader:
|
|
| 789 |
SET n.title=$title, n.circular_number=$cno,
|
| 790 |
n.date=$date, n.ministry=$ministry,
|
| 791 |
n.subject=$subject, n.source=$source,
|
| 792 |
-
n.dataset=
|
| 793 |
-
""",
|
| 794 |
-
|
| 795 |
-
|
| 796 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 797 |
count += 1
|
| 798 |
except Exception as e:
|
| 799 |
logger.warning(f"[Loader] VigilanceCircular failed: {e}")
|
|
@@ -801,6 +739,233 @@ class GraphLoader:
|
|
| 801 |
logger.success(f"[Loader] Loaded {count} CVC circulars")
|
| 802 |
return count
|
| 803 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 804 |
def close(self):
|
| 805 |
if self.driver:
|
| 806 |
self.driver.close()
|
|
@@ -809,10 +974,8 @@ class GraphLoader:
|
|
| 809 |
|
| 810 |
if __name__ == "__main__":
|
| 811 |
parser = argparse.ArgumentParser(description="BharatGraph Graph Loader")
|
| 812 |
-
parser.add_argument("--dry-run", action="store_true"
|
| 813 |
-
|
| 814 |
-
parser.add_argument("--file", type=str, default=None,
|
| 815 |
-
help="Path to pipeline output JSON file")
|
| 816 |
args = parser.parse_args()
|
| 817 |
|
| 818 |
print("=" * 55)
|
|
@@ -824,28 +987,21 @@ if __name__ == "__main__":
|
|
| 824 |
if args.file:
|
| 825 |
results = loader.load_from_pipeline_output(args.file)
|
| 826 |
else:
|
| 827 |
-
# Find latest pipeline output
|
| 828 |
import glob
|
| 829 |
files = sorted(glob.glob("data/processed/pipeline_*.json"))
|
| 830 |
if files:
|
| 831 |
-
|
| 832 |
-
print(f"\nLoading latest pipeline output: {latest}")
|
| 833 |
-
results = loader.load_from_pipeline_output(latest)
|
| 834 |
else:
|
| 835 |
-
print("\nNo pipeline output found.")
|
| 836 |
-
print("Run first: python -m processing.pipeline --scrapers cag,gem,pib")
|
| 837 |
results = {}
|
| 838 |
|
| 839 |
loader.close()
|
| 840 |
-
|
| 841 |
-
print(f"\n{'='*55}")
|
| 842 |
-
print("LOAD SUMMARY")
|
| 843 |
-
print(f"{'='*55}")
|
| 844 |
s = loader.stats
|
| 845 |
-
print(f"
|
| 846 |
-
print(f" Nodes
|
| 847 |
-
print(f"
|
| 848 |
-
print(f"
|
|
|
|
| 849 |
if args.dry_run:
|
| 850 |
-
print("
|
| 851 |
-
print(
|
|
|
|
| 1 |
"""
|
| 2 |
BharatGraph - Neo4j Graph Loader
|
| 3 |
Loads cleaned + resolved data from Phase 2 pipeline into Neo4j.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
| 5 |
|
| 6 |
+
import os, sys, json, hashlib, argparse
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
from datetime import datetime
|
| 8 |
from loguru import logger
|
| 9 |
|
| 10 |
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
| 11 |
from graph.schema import SETUP_QUERIES, NODE_SCHEMAS
|
| 12 |
|
| 13 |
|
| 14 |
+
# BUG-19 FIX: upgraded from MD5 (16-char) to SHA-256 (20-char) for lower collision
|
| 15 |
+
# risk across large datasets.
|
| 16 |
def make_id(*parts) -> str:
|
| 17 |
+
"""Create a stable SHA-256 derived ID from multiple string parts."""
|
| 18 |
combined = "|".join(str(p).lower().strip() for p in parts)
|
| 19 |
+
return hashlib.sha256(combined.encode()).hexdigest()[:20]
|
| 20 |
|
| 21 |
|
| 22 |
class GraphLoader:
|
| 23 |
"""
|
| 24 |
Loads BharatGraph data into Neo4j.
|
| 25 |
+
BUG-21 FIX: __init__ now accepts an optional pre-built driver so admin.py
|
| 26 |
+
can pass the FastAPI dependency driver instead of opening a second connection.
|
| 27 |
"""
|
| 28 |
|
| 29 |
+
def __init__(self, dry_run: bool = False, driver=None):
|
| 30 |
self.dry_run = dry_run
|
| 31 |
+
self.driver = driver # BUG-21 FIX: accept injected driver
|
| 32 |
self.stats = {
|
| 33 |
+
"nodes_created": 0,
|
| 34 |
+
"nodes_merged": 0,
|
| 35 |
+
"rels_created": 0,
|
| 36 |
+
"errors": 0,
|
| 37 |
}
|
| 38 |
if dry_run:
|
| 39 |
+
logger.info("[Loader] DRY RUN mode — no data written to Neo4j")
|
| 40 |
+
elif driver is None:
|
| 41 |
self._connect()
|
| 42 |
|
| 43 |
def _connect(self):
|
|
|
|
| 48 |
load_dotenv()
|
| 49 |
|
| 50 |
uri = os.getenv("NEO4J_URI", "")
|
| 51 |
+
user = os.getenv("NEO4J_USER", "neo4j")
|
| 52 |
pwd = os.getenv("NEO4J_PASSWORD", "")
|
| 53 |
|
| 54 |
if not pwd:
|
|
|
|
| 58 |
self.driver = GraphDatabase.driver(uri, auth=(user, pwd))
|
| 59 |
self.driver.verify_connectivity()
|
| 60 |
logger.success(f"[Loader] Connected to Neo4j: {uri[:40]}...")
|
|
|
|
| 61 |
except ImportError:
|
| 62 |
+
logger.error("[Loader] neo4j package not installed.")
|
| 63 |
raise
|
| 64 |
except Exception as e:
|
| 65 |
logger.error(f"[Loader] Connection failed: {e}")
|
|
|
|
| 71 |
logger.info(f"[Loader] DRY RUN: would run {len(SETUP_QUERIES)} setup queries")
|
| 72 |
return
|
| 73 |
with self.driver.session() as session:
|
| 74 |
+
# BUG-3 FIX: fulltext index now covers all 20 node types (was 8).
|
| 75 |
try:
|
| 76 |
session.run(
|
| 77 |
"CALL db.index.fulltext.createNodeIndex("
|
| 78 |
"'globalSearch',"
|
| 79 |
"['Politician','Company','Contract','AuditReport',"
|
| 80 |
+
" 'Scheme','Ministry','Party','PressRelease',"
|
| 81 |
+
" 'NGO','ElectoralBond','InsolvencyOrder','Tender',"
|
| 82 |
+
" 'RegulatoryOrder','EnforcementAction',"
|
| 83 |
+
" 'ParliamentQuestion','VigilanceCircular',"
|
| 84 |
+
" 'ICIJEntity','SanctionedEntity','CourtCase','LocalBody'],"
|
| 85 |
"['name','title','aliases','item_desc','product','buyer_org',"
|
| 86 |
+
" 'cin','ministry','summary','seller_name','ngo_name',"
|
| 87 |
+
" 'purchaser_name','redeemed_by','company_name','accused',"
|
| 88 |
+
" 'entity_name','subject','jurisdiction']"
|
| 89 |
")"
|
| 90 |
)
|
| 91 |
+
logger.info("[Loader] Full-text index created or verified (20 types)")
|
| 92 |
except Exception as e:
|
| 93 |
logger.debug(f"[Loader] Full-text index note: {e}")
|
| 94 |
|
|
|
|
| 95 |
for query in SETUP_QUERIES:
|
| 96 |
try:
|
| 97 |
session.run(query)
|
|
|
|
| 100 |
|
| 101 |
logger.success("[Loader] Schema constraints and indexes ready")
|
| 102 |
|
| 103 |
+
def _run(self, query: str, params: dict = None, **kwargs):
|
| 104 |
+
"""
|
| 105 |
+
Execute a single Cypher query.
|
| 106 |
+
BUG-17 FIX: now accepts both calling styles:
|
| 107 |
+
- old style: self._run(query, {"key": val}) ← positional dict
|
| 108 |
+
- new style: self._run(query, key=val, ...) ← keyword args
|
| 109 |
+
Both are merged so existing callers and the 8 new loaders both work.
|
| 110 |
+
"""
|
| 111 |
if self.dry_run:
|
| 112 |
logger.debug(f"[Loader] DRY RUN query: {query[:80]}...")
|
| 113 |
return None
|
| 114 |
+
# Merge positional dict with any keyword args
|
| 115 |
+
all_params = {**(params or {}), **kwargs}
|
| 116 |
try:
|
| 117 |
with self.driver.session() as session:
|
| 118 |
+
return session.run(query, all_params)
|
|
|
|
| 119 |
except Exception as e:
|
| 120 |
logger.error(f"[Loader] Query failed: {e}\nQuery: {query[:100]}")
|
| 121 |
self.stats["errors"] += 1
|
|
|
|
| 124 |
# ── Node loaders ──────────────────────────────────────
|
| 125 |
|
| 126 |
def load_politicians(self, records: list) -> int:
|
|
|
|
| 127 |
count = 0
|
| 128 |
for r in records:
|
| 129 |
name = r.get("name", "").strip()
|
|
|
|
| 160 |
}
|
| 161 |
self._run(query, params)
|
| 162 |
count += 1
|
|
|
|
|
|
|
| 163 |
party = r.get("party", "").strip()
|
| 164 |
if party:
|
| 165 |
self._create_party_and_link(node_id, party)
|
|
|
|
| 169 |
return count
|
| 170 |
|
| 171 |
def _create_party_and_link(self, politician_id: str, party_name: str):
|
|
|
|
| 172 |
party_id = make_id(party_name)
|
| 173 |
query = """
|
| 174 |
MERGE (party:Party {id: $party_id})
|
|
|
|
| 178 |
MERGE (p)-[:MEMBER_OF]->(party)
|
| 179 |
"""
|
| 180 |
self._run(query, {
|
| 181 |
+
"party_id": party_id,
|
| 182 |
"party_name": party_name,
|
| 183 |
+
"pol_id": politician_id,
|
| 184 |
})
|
| 185 |
|
| 186 |
def load_companies(self, records: list) -> int:
|
|
|
|
| 187 |
count = 0
|
| 188 |
for r in records:
|
| 189 |
name = r.get("name", "").strip()
|
|
|
|
| 222 |
return count
|
| 223 |
|
| 224 |
def load_contracts(self, records: list) -> int:
|
|
|
|
| 225 |
count = 0
|
| 226 |
for r in records:
|
| 227 |
order_id = r.get("order_id", "").strip()
|
|
|
|
| 229 |
if not order_id:
|
| 230 |
continue
|
| 231 |
node_id = make_id(order_id)
|
|
|
|
| 232 |
query = """
|
| 233 |
MERGE (c:Contract {id: $id})
|
| 234 |
SET c.order_id = $order_id,
|
|
|
|
| 255 |
}
|
| 256 |
self._run(query, params)
|
| 257 |
count += 1
|
|
|
|
|
|
|
| 258 |
if seller:
|
| 259 |
+
self._link_company_to_contract(seller, node_id, float(r.get("amount_crore", 0) or 0))
|
|
|
|
|
|
|
|
|
|
| 260 |
buyer = r.get("buyer_org", "").strip()
|
| 261 |
if buyer:
|
| 262 |
self._link_contract_to_ministry(node_id, buyer)
|
|
|
|
| 265 |
self.stats["nodes_created"] += count
|
| 266 |
return count
|
| 267 |
|
| 268 |
+
def _link_company_to_contract(self, seller_name, contract_id, amount):
|
|
|
|
|
|
|
| 269 |
company_id = make_id(seller_name)
|
| 270 |
+
self._run("""
|
| 271 |
MERGE (co:Company {id: $co_id})
|
| 272 |
SET co.name = $seller_name
|
| 273 |
WITH co
|
| 274 |
MATCH (c:Contract {id: $contract_id})
|
| 275 |
MERGE (co)-[r:WON_CONTRACT]->(c)
|
| 276 |
SET r.amount_crore = $amount
|
| 277 |
+
""", {"co_id": company_id, "seller_name": seller_name,
|
| 278 |
+
"contract_id": contract_id, "amount": amount})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
self.stats["rels_created"] += 1
|
| 280 |
|
| 281 |
+
def _link_contract_to_ministry(self, contract_id, ministry_name):
|
|
|
|
| 282 |
ministry_id = make_id(ministry_name)
|
| 283 |
+
self._run("""
|
| 284 |
MERGE (m:Ministry {id: $m_id})
|
| 285 |
SET m.name = $m_name
|
| 286 |
WITH m
|
| 287 |
MATCH (c:Contract {id: $contract_id})
|
| 288 |
MERGE (c)-[:AWARDED_BY]->(m)
|
| 289 |
+
""", {"m_id": ministry_id, "m_name": ministry_name, "contract_id": contract_id})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
self.stats["rels_created"] += 1
|
| 291 |
|
| 292 |
def load_audit_reports(self, records: list) -> int:
|
|
|
|
| 293 |
count = 0
|
| 294 |
for r in records:
|
| 295 |
title = r.get("title", "").strip()
|
| 296 |
if not title:
|
| 297 |
continue
|
| 298 |
node_id = make_id(title, r.get("url", ""))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
params = {
|
| 300 |
"id": node_id,
|
| 301 |
"title": title,
|
|
|
|
| 310 |
"source": r.get("_source", "cag"),
|
| 311 |
"scraped_at": r.get("scraped_at", datetime.now().isoformat()),
|
| 312 |
}
|
| 313 |
+
self._run("""
|
| 314 |
+
MERGE (a:AuditReport {id: $id})
|
| 315 |
+
SET a.title=$title, a.url=$url, a.year=$year, a.state=$state,
|
| 316 |
+
a.scheme=$scheme, a.amount_crore=$amount,
|
| 317 |
+
a.irregularity_type=$irr_type, a.finding=$finding,
|
| 318 |
+
a.alert_keywords=$keywords, a.source=$source, a.scraped_at=$scraped_at
|
| 319 |
+
""", params)
|
| 320 |
count += 1
|
|
|
|
|
|
|
| 321 |
scheme = r.get("scheme", "").strip()
|
| 322 |
if scheme:
|
| 323 |
+
self._link_audit_to_scheme(node_id, scheme, float(r.get("amount_crore", 0) or 0))
|
|
|
|
| 324 |
|
| 325 |
logger.success(f"[Loader] Audit reports loaded: {count}")
|
| 326 |
self.stats["nodes_created"] += count
|
| 327 |
return count
|
| 328 |
|
| 329 |
+
def _link_audit_to_scheme(self, audit_id, scheme_name, amount):
|
|
|
|
|
|
|
| 330 |
scheme_id = make_id(scheme_name)
|
| 331 |
+
self._run("""
|
| 332 |
MERGE (s:Scheme {id: $s_id})
|
| 333 |
SET s.name = $s_name
|
| 334 |
WITH s
|
| 335 |
MATCH (a:AuditReport {id: $audit_id})
|
| 336 |
MERGE (a)-[r:FLAGS]->(s)
|
| 337 |
SET r.amount_crore = $amount
|
| 338 |
+
""", {"s_id": scheme_id, "s_name": scheme_name, "audit_id": audit_id, "amount": amount})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 339 |
self.stats["rels_created"] += 1
|
| 340 |
|
| 341 |
def load_press_releases(self, records: list) -> int:
|
|
|
|
| 342 |
count = 0
|
| 343 |
for r in records:
|
| 344 |
title = r.get("title", "").strip()
|
| 345 |
if not title:
|
| 346 |
continue
|
| 347 |
node_id = make_id(title, r.get("link", ""))
|
| 348 |
+
self._run("""
|
| 349 |
MERGE (pr:PressRelease {id: $id})
|
| 350 |
+
SET pr.title=$title, pr.link=$link, pr.published=$published,
|
| 351 |
+
pr.alert_keywords=$keywords, pr.source=$source, pr.scraped_at=$scraped_at
|
| 352 |
+
""", {
|
| 353 |
+
"id": node_id, "title": title, "link": r.get("link", ""),
|
| 354 |
+
"published": r.get("published", ""),
|
| 355 |
+
"keywords": str(r.get("alert_keywords", [])),
|
| 356 |
+
"source": r.get("_source", "pib"),
|
| 357 |
+
"scraped_at":r.get("scraped_at", datetime.now().isoformat()),
|
| 358 |
+
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 359 |
count += 1
|
| 360 |
|
| 361 |
logger.success(f"[Loader] Press releases loaded: {count}")
|
|
|
|
| 363 |
return count
|
| 364 |
|
| 365 |
def load_politician_company_links(self, matches: list) -> int:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
count = 0
|
| 367 |
for match in matches:
|
| 368 |
pol_name = match.get("name_a", match.get("politician_name", "")).strip()
|
|
|
|
| 370 |
score = match.get("score", 0.0)
|
| 371 |
if not pol_name or not co_name:
|
| 372 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
link_query = """
|
| 374 |
OPTIONAL MATCH (existing_p:Politician)
|
| 375 |
WHERE toLower(existing_p.name) = toLower($pol_name)
|
|
|
|
| 405 |
return count
|
| 406 |
|
| 407 |
def load_from_pipeline_output(self, filepath: str) -> dict:
|
| 408 |
+
"""Load everything from a pipeline JSON output file."""
|
|
|
|
|
|
|
|
|
|
| 409 |
logger.info(f"[Loader] Loading from: {filepath}")
|
| 410 |
with open(filepath, encoding="utf-8") as f:
|
| 411 |
data = json.load(f)
|
|
|
|
| 416 |
self.setup_schema()
|
| 417 |
|
| 418 |
results = {}
|
| 419 |
+
if raw.get("myneta"): results["politicians"] = self.load_politicians(raw["myneta"])
|
| 420 |
+
if raw.get("mca"): results["companies"] = self.load_companies(raw["mca"])
|
| 421 |
+
if raw.get("gem"): results["contracts"] = self.load_contracts(raw["gem"])
|
| 422 |
+
if raw.get("cag"): results["audit_reports"] = self.load_audit_reports(raw["cag"])
|
| 423 |
+
if raw.get("pib"): results["press_releases"] = self.load_press_releases(raw["pib"])
|
| 424 |
+
if links: results["director_of_links"] = self.load_politician_company_links(links)
|
| 425 |
+
if raw.get("sebi"): results["regulatory_orders"] = self.load_regulatory_orders(raw["sebi"])
|
| 426 |
+
if raw.get("ed"): results["enforcement_actions"] = self.load_enforcement_actions(raw["ed"])
|
| 427 |
+
if raw.get("electoral_bond"):results["electoral_bonds"] = self.load_electoral_bonds(raw["electoral_bond"])
|
| 428 |
+
if raw.get("ibbi"): results["insolvency_orders"] = self.load_insolvency_orders(raw["ibbi"])
|
| 429 |
+
if raw.get("ngo_darpan"): results["ngos"] = self.load_ngos(raw["ngo_darpan"])
|
| 430 |
+
if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
|
| 431 |
+
if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
|
| 432 |
+
if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
|
| 433 |
+
# BUG-2 FIX: 7 previously missing loaders — these datasets were scraped but
|
| 434 |
+
# silently dropped because load_from_pipeline_output never called them.
|
| 435 |
+
if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
|
| 436 |
+
if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
|
| 437 |
+
if raw.get("njdg"): results["court_cases"] = self.load_court_cases(raw["njdg"])
|
| 438 |
+
if raw.get("lgd"): results["local_bodies"] = self.load_local_bodies(raw["lgd"])
|
| 439 |
+
if raw.get("ncrb"): results["crime_reports"] = self.load_crime_reports(raw["ncrb"])
|
| 440 |
+
if raw.get("wikidata"): results["wikidata_enrichments"]= self.load_wikidata_enrichments(raw["wikidata"])
|
| 441 |
+
if raw.get("datagov"): results["datagov_documents"] = self.load_datagov_documents(raw["datagov"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 442 |
|
| 443 |
logger.success(f"[Loader] Load complete. Stats: {self.stats}")
|
| 444 |
return {**results, "stats": self.stats}
|
| 445 |
|
| 446 |
+
# ── Phase 28 loaders (8 datasets) ────────────────────────────────────────
|
|
|
|
| 447 |
|
| 448 |
def load_regulatory_orders(self, records: list) -> int:
|
|
|
|
| 449 |
count = 0
|
| 450 |
for r in records:
|
| 451 |
title = (r.get("title") or "").strip()
|
|
|
|
| 456 |
MERGE (n:RegulatoryOrder {id:$id})
|
| 457 |
SET n.title=$title, n.url=$url, n.order_type=$order_type,
|
| 458 |
n.entity_name=$entity_name, n.violation=$violation,
|
| 459 |
+
n.source=$source, n.dataset='sebi', n.scraped_at=$scraped_at
|
| 460 |
+
""", {
|
| 461 |
+
"id": node_id, "title": title, "url": r.get("url",""),
|
| 462 |
+
"order_type": r.get("order_type",""),
|
| 463 |
+
"entity_name": r.get("entity_name", r.get("accused","")),
|
| 464 |
+
"violation": r.get("violation",""),
|
| 465 |
+
"source": r.get("source","SEBI"),
|
| 466 |
+
"scraped_at": r.get("scraped_at",""),
|
| 467 |
+
})
|
| 468 |
count += 1
|
| 469 |
except Exception as e:
|
| 470 |
logger.warning(f"[Loader] RegulatoryOrder failed: {e}")
|
|
|
|
| 473 |
return count
|
| 474 |
|
| 475 |
def load_enforcement_actions(self, records: list) -> int:
|
|
|
|
| 476 |
count = 0
|
| 477 |
for r in records:
|
| 478 |
title = (r.get("title") or "").strip()
|
|
|
|
| 484 |
SET n.title=$title, n.url=$url, n.date=$date,
|
| 485 |
n.amount_crore=$amount_crore, n.case_type=$case_type,
|
| 486 |
n.accused=$accused, n.source=$source,
|
| 487 |
+
n.dataset='ed', n.scraped_at=$scraped_at
|
| 488 |
+
""", {
|
| 489 |
+
"id": node_id, "title": title, "url": r.get("url",""),
|
| 490 |
+
"date": r.get("date",""),
|
| 491 |
+
"amount_crore": float(r.get("amount_crore",0) or 0),
|
| 492 |
+
"case_type": r.get("case_type",""),
|
| 493 |
+
"accused": r.get("accused",""),
|
| 494 |
+
"source": r.get("source","ED"),
|
| 495 |
+
"scraped_at": r.get("scraped_at",""),
|
| 496 |
+
})
|
| 497 |
count += 1
|
| 498 |
accused = r.get("accused","").strip()
|
| 499 |
if accused:
|
|
|
|
| 502 |
WHERE toLower(p.name) CONTAINS toLower($accused)
|
| 503 |
MATCH (n:EnforcementAction {id:$id})
|
| 504 |
MERGE (p)-[:SUBJECT_OF]->(n)
|
| 505 |
+
""", {"accused": accused, "id": node_id})
|
| 506 |
except Exception as e:
|
| 507 |
logger.warning(f"[Loader] EnforcementAction failed: {e}")
|
| 508 |
self.stats["enforcement_actions"] = count
|
|
|
|
| 510 |
return count
|
| 511 |
|
| 512 |
def load_electoral_bonds(self, records: list) -> int:
|
|
|
|
| 513 |
count = 0
|
| 514 |
for r in records:
|
| 515 |
purchaser = (r.get("purchaser_name") or "").strip()
|
|
|
|
| 521 |
SET n.bond_number=$bond_no, n.purchaser_name=$purchaser,
|
| 522 |
n.denomination_crore=$denom, n.purchase_date=$p_date,
|
| 523 |
n.redemption_date=$r_date, n.redeemed_by=$redeemer,
|
| 524 |
+
n.source=$source, n.dataset='electoral_bond',
|
| 525 |
n.scraped_at=$scraped_at
|
| 526 |
+
""", {
|
| 527 |
+
"id": node_id, "bond_no": r.get("bond_number",""),
|
| 528 |
+
"purchaser": purchaser,
|
| 529 |
+
"denom": float(r.get("denomination_crore",0) or 0),
|
| 530 |
+
"p_date": r.get("purchase_date",""),
|
| 531 |
+
"r_date": r.get("redemption_date",""),
|
| 532 |
+
"redeemer": r.get("redeemed_by",""),
|
| 533 |
+
"source": r.get("source","ECI"),
|
| 534 |
+
"scraped_at":r.get("scraped_at",""),
|
| 535 |
+
})
|
| 536 |
count += 1
|
| 537 |
redeemer = r.get("redeemed_by","").strip()
|
| 538 |
if purchaser and redeemer:
|
| 539 |
self._run("""
|
| 540 |
MERGE (c:Company {id:$co_id})
|
| 541 |
+
ON CREATE SET c.name=$purchaser, c.source='electoral_bond'
|
| 542 |
MERGE (p:Party {id:$pt_id})
|
| 543 |
+
ON CREATE SET p.name=$redeemer, p.source='electoral_bond'
|
| 544 |
MATCH (b:ElectoralBond {id:$b_id})
|
| 545 |
MERGE (c)-[:DONATED_VIA]->(b)-[:REDEEMED_BY]->(p)
|
| 546 |
+
""", {
|
| 547 |
+
"co_id": make_id(purchaser), "purchaser": purchaser,
|
| 548 |
+
"pt_id": make_id(redeemer), "redeemer": redeemer,
|
| 549 |
+
"b_id": node_id,
|
| 550 |
+
})
|
| 551 |
except Exception as e:
|
| 552 |
logger.warning(f"[Loader] ElectoralBond failed: {e}")
|
| 553 |
self.stats["electoral_bonds"] = count
|
|
|
|
| 555 |
return count
|
| 556 |
|
| 557 |
def load_insolvency_orders(self, records: list) -> int:
|
|
|
|
| 558 |
count = 0
|
| 559 |
for r in records:
|
| 560 |
company = (r.get("company_name") or "").strip()
|
|
|
|
| 566 |
SET n.company_name=$company, n.cin=$cin,
|
| 567 |
n.process_type=$process_type, n.admitted_date=$admitted_date,
|
| 568 |
n.status=$status, n.admitted_claims=$claims,
|
| 569 |
+
n.source=$source, n.dataset='ibbi', n.scraped_at=$scraped_at
|
| 570 |
+
""", {
|
| 571 |
+
"id": node_id, "company": company, "cin": r.get("cin",""),
|
| 572 |
+
"process_type": r.get("process_type",""),
|
| 573 |
+
"admitted_date": r.get("admitted_date",""),
|
| 574 |
+
"status": r.get("status",""),
|
| 575 |
+
"claims": float(r.get("admitted_claims",0) or 0),
|
| 576 |
+
"source": r.get("source","IBBI"),
|
| 577 |
+
"scraped_at": r.get("scraped_at",""),
|
| 578 |
+
})
|
| 579 |
count += 1
|
| 580 |
self._run("""
|
| 581 |
MATCH (c:Company)
|
| 582 |
WHERE toLower(c.name) CONTAINS toLower($company)
|
| 583 |
MATCH (n:InsolvencyOrder {id:$id})
|
| 584 |
MERGE (c)-[:HAS_INSOLVENCY]->(n)
|
| 585 |
+
""", {"company": company, "id": node_id})
|
| 586 |
except Exception as e:
|
| 587 |
logger.warning(f"[Loader] InsolvencyOrder failed: {e}")
|
| 588 |
self.stats["insolvency_orders"] = count
|
|
|
|
| 590 |
return count
|
| 591 |
|
| 592 |
def load_ngos(self, records: list) -> int:
|
|
|
|
| 593 |
count = 0
|
| 594 |
for r in records:
|
| 595 |
name = (r.get("ngo_name") or "").strip()
|
|
|
|
| 603 |
n.registration_type=$reg_type, n.year_of_reg=$year,
|
| 604 |
n.key_issues=$issues, n.csr_receipts=$csr,
|
| 605 |
n.govt_grants=$grants, n.source=$source,
|
| 606 |
+
n.dataset='ngo_darpan', n.scraped_at=$scraped_at
|
| 607 |
+
""", {
|
| 608 |
+
"id": node_id, "name": name,
|
| 609 |
+
"darpan_id": r.get("darpan_id",""),
|
| 610 |
+
"state": r.get("state",""),
|
| 611 |
+
"district": r.get("district",""),
|
| 612 |
+
"reg_type": r.get("registration_type",""),
|
| 613 |
+
"year": str(r.get("year_of_reg","")),
|
| 614 |
+
"issues": str(r.get("key_issues",[])),
|
| 615 |
+
"csr": float(r.get("csr_receipts",0) or 0),
|
| 616 |
+
"grants": float(r.get("govt_grants",0) or 0),
|
| 617 |
+
"source": r.get("source","NGO Darpan"),
|
| 618 |
+
"scraped_at":r.get("scraped_at",""),
|
| 619 |
+
})
|
| 620 |
count += 1
|
| 621 |
except Exception as e:
|
| 622 |
logger.warning(f"[Loader] NGO failed: {e}")
|
|
|
|
| 625 |
return count
|
| 626 |
|
| 627 |
def load_tenders(self, records: list) -> int:
|
|
|
|
| 628 |
count = 0
|
| 629 |
for r in records:
|
| 630 |
title = (r.get("title") or "").strip()
|
|
|
|
| 639 |
n.bid_end_date=$bid_end, n.status=$status,
|
| 640 |
n.awarded_to=$awarded_to, n.awarded_value=$awarded_val,
|
| 641 |
n.single_bid=$single_bid, n.source=$source,
|
| 642 |
+
n.dataset='cppp', n.scraped_at=$scraped_at
|
| 643 |
+
""", {
|
| 644 |
+
"id": node_id, "tender_id": r.get("tender_id",""),
|
| 645 |
+
"title": title,
|
| 646 |
+
"ministry": r.get("ministry",""),
|
| 647 |
+
"dept": r.get("department",""),
|
| 648 |
+
"value": float(r.get("estimated_crore",0) or 0),
|
| 649 |
+
"t_type": r.get("tender_type",""),
|
| 650 |
+
"bid_end": r.get("bid_submission_end",""),
|
| 651 |
+
"status": r.get("status",""),
|
| 652 |
+
"awarded_to": r.get("awarded_to",""),
|
| 653 |
+
"awarded_val":float(r.get("awarded_value",0) or 0),
|
| 654 |
+
"single_bid": bool(r.get("single_bid",False)),
|
| 655 |
+
"source": r.get("source","CPPP"),
|
| 656 |
+
"scraped_at": r.get("scraped_at",""),
|
| 657 |
+
})
|
| 658 |
count += 1
|
| 659 |
ministry = r.get("ministry","").strip()
|
| 660 |
if ministry:
|
| 661 |
self._run("""
|
| 662 |
MERGE (m:Ministry {id:$m_id})
|
| 663 |
+
ON CREATE SET m.name=$ministry, m.source='cppp'
|
| 664 |
MATCH (t:Tender {id:$t_id})
|
| 665 |
MERGE (m)-[:ISSUED_TENDER]->(t)
|
| 666 |
+
""", {"m_id": make_id(ministry), "ministry": ministry, "t_id": node_id})
|
| 667 |
if r.get("single_bid"):
|
| 668 |
self._run("MATCH (t:Tender {id:$id}) SET t.risk_flag='SINGLE_BID'",
|
| 669 |
+
{"id": node_id})
|
| 670 |
except Exception as e:
|
| 671 |
logger.warning(f"[Loader] Tender failed: {e}")
|
| 672 |
self.stats["tenders"] = count
|
|
|
|
| 674 |
return count
|
| 675 |
|
| 676 |
def load_parliament_questions(self, records: list) -> int:
|
|
|
|
| 677 |
count = 0
|
| 678 |
for r in records:
|
| 679 |
subject = (r.get("subject") or "").strip()
|
|
|
|
| 685 |
SET n.subject=$subject, n.question_number=$qno,
|
| 686 |
n.member_name=$member, n.ministry=$ministry,
|
| 687 |
n.session=$session, n.source=$source,
|
| 688 |
+
n.dataset='loksabha', n.scraped_at=$scraped_at
|
| 689 |
+
""", {
|
| 690 |
+
"id": node_id, "subject": subject,
|
| 691 |
+
"qno": r.get("question_number",""),
|
| 692 |
+
"member": r.get("member_name",""),
|
| 693 |
+
"ministry": r.get("ministry",""),
|
| 694 |
+
"session": r.get("session",""),
|
| 695 |
+
"source": r.get("source","Lok Sabha"),
|
| 696 |
+
"scraped_at":r.get("scraped_at",""),
|
| 697 |
+
})
|
| 698 |
count += 1
|
| 699 |
member = r.get("member_name","").strip()
|
| 700 |
if member:
|
|
|
|
| 703 |
WHERE toLower(p.name) CONTAINS toLower($member)
|
| 704 |
MATCH (q:ParliamentQuestion {id:$id})
|
| 705 |
MERGE (p)-[:ASKED_QUESTION]->(q)
|
| 706 |
+
""", {"member": member, "id": node_id})
|
| 707 |
except Exception as e:
|
| 708 |
logger.warning(f"[Loader] ParliamentQuestion failed: {e}")
|
| 709 |
self.stats["parliament_questions"] = count
|
|
|
|
| 711 |
return count
|
| 712 |
|
| 713 |
def load_vigilance_circulars(self, records: list) -> int:
|
|
|
|
| 714 |
count = 0
|
| 715 |
for r in records:
|
| 716 |
title = (r.get("title") or "").strip()
|
|
|
|
| 722 |
SET n.title=$title, n.circular_number=$cno,
|
| 723 |
n.date=$date, n.ministry=$ministry,
|
| 724 |
n.subject=$subject, n.source=$source,
|
| 725 |
+
n.dataset='cvc', n.scraped_at=$scraped_at
|
| 726 |
+
""", {
|
| 727 |
+
"id": node_id, "title": title,
|
| 728 |
+
"cno": r.get("circular_number",""),
|
| 729 |
+
"date": r.get("date",""),
|
| 730 |
+
"ministry": r.get("ministry",""),
|
| 731 |
+
"subject": r.get("subject",""),
|
| 732 |
+
"source": r.get("source","CVC"),
|
| 733 |
+
"scraped_at":r.get("scraped_at",""),
|
| 734 |
+
})
|
| 735 |
count += 1
|
| 736 |
except Exception as e:
|
| 737 |
logger.warning(f"[Loader] VigilanceCircular failed: {e}")
|
|
|
|
| 739 |
logger.success(f"[Loader] Loaded {count} CVC circulars")
|
| 740 |
return count
|
| 741 |
|
| 742 |
+
# ── BUG-2 FIX: 7 NEW loaders — were scraped but never loaded ─────────────
|
| 743 |
+
|
| 744 |
+
def load_icij_entities(self, records: list) -> int:
|
| 745 |
+
"""ICIJ Offshore Leaks entities → ICIJEntity nodes."""
|
| 746 |
+
count = 0
|
| 747 |
+
for r in records:
|
| 748 |
+
name = (r.get("name") or r.get("entity_name") or "").strip()
|
| 749 |
+
if not name: continue
|
| 750 |
+
node_id = make_id(f"icij_{r.get('node_id', name)}")
|
| 751 |
+
try:
|
| 752 |
+
self._run("""
|
| 753 |
+
MERGE (n:ICIJEntity {id:$id})
|
| 754 |
+
SET n.name=$name, n.entity_type=$entity_type,
|
| 755 |
+
n.jurisdiction=$jurisdiction, n.linked_to=$linked_to,
|
| 756 |
+
n.dataset_name=$dataset_name, n.source='ICIJ',
|
| 757 |
+
n.dataset='icij', n.scraped_at=$scraped_at
|
| 758 |
+
""", {
|
| 759 |
+
"id": node_id, "name": name,
|
| 760 |
+
"entity_type": r.get("entity_type", r.get("type","")),
|
| 761 |
+
"jurisdiction": r.get("jurisdiction",""),
|
| 762 |
+
"linked_to": str(r.get("linked_to",[])),
|
| 763 |
+
"dataset_name": r.get("sourceID", r.get("dataset_name","")),
|
| 764 |
+
"scraped_at": r.get("scraped_at",""),
|
| 765 |
+
})
|
| 766 |
+
count += 1
|
| 767 |
+
# Cross-link to Indian politicians/companies by name similarity
|
| 768 |
+
self._run("""
|
| 769 |
+
MATCH (p:Politician)
|
| 770 |
+
WHERE toLower(p.name) CONTAINS toLower($name)
|
| 771 |
+
MATCH (n:ICIJEntity {id:$id})
|
| 772 |
+
MERGE (p)-[:ASSOCIATED_WITH]->(n)
|
| 773 |
+
""", {"name": name, "id": node_id})
|
| 774 |
+
except Exception as e:
|
| 775 |
+
logger.warning(f"[Loader] ICIJEntity failed: {e}")
|
| 776 |
+
self.stats["icij_entities"] = count
|
| 777 |
+
logger.success(f"[Loader] Loaded {count} ICIJ entities")
|
| 778 |
+
return count
|
| 779 |
+
|
| 780 |
+
def load_sanctioned_entities(self, records: list) -> int:
|
| 781 |
+
"""OpenSanctions → SanctionedEntity nodes."""
|
| 782 |
+
count = 0
|
| 783 |
+
for r in records:
|
| 784 |
+
name = (r.get("name") or r.get("caption") or "").strip()
|
| 785 |
+
if not name: continue
|
| 786 |
+
node_id = make_id(f"sanction_{r.get('id', name)}")
|
| 787 |
+
try:
|
| 788 |
+
self._run("""
|
| 789 |
+
MERGE (n:SanctionedEntity {id:$id})
|
| 790 |
+
SET n.name=$name, n.schema_type=$schema_type,
|
| 791 |
+
n.topics=$topics, n.countries=$countries,
|
| 792 |
+
n.sanctions_list=$sanctions_list,
|
| 793 |
+
n.source='OpenSanctions', n.dataset='opensanctions',
|
| 794 |
+
n.scraped_at=$scraped_at
|
| 795 |
+
""", {
|
| 796 |
+
"id": node_id, "name": name,
|
| 797 |
+
"schema_type": r.get("schema",""),
|
| 798 |
+
"topics": str(r.get("topics",[])),
|
| 799 |
+
"countries": str(r.get("countries",[])),
|
| 800 |
+
"sanctions_list":r.get("datasets",""),
|
| 801 |
+
"scraped_at": r.get("scraped_at",""),
|
| 802 |
+
})
|
| 803 |
+
count += 1
|
| 804 |
+
self._run("""
|
| 805 |
+
MATCH (p:Politician)
|
| 806 |
+
WHERE toLower(p.name) CONTAINS toLower($name)
|
| 807 |
+
MATCH (n:SanctionedEntity {id:$id})
|
| 808 |
+
MERGE (p)-[:ASSOCIATED_WITH]->(n)
|
| 809 |
+
""", {"name": name, "id": node_id})
|
| 810 |
+
except Exception as e:
|
| 811 |
+
logger.warning(f"[Loader] SanctionedEntity failed: {e}")
|
| 812 |
+
self.stats["sanctioned_entities"] = count
|
| 813 |
+
logger.success(f"[Loader] Loaded {count} sanctioned entities")
|
| 814 |
+
return count
|
| 815 |
+
|
| 816 |
+
def load_court_cases(self, records: list) -> int:
|
| 817 |
+
"""NJDG court pendency stats → CourtCase nodes."""
|
| 818 |
+
count = 0
|
| 819 |
+
for r in records:
|
| 820 |
+
court = (r.get("court_name") or r.get("state") or "").strip()
|
| 821 |
+
if not court: continue
|
| 822 |
+
node_id = make_id(f"njdg_{court}_{r.get('year','')}")
|
| 823 |
+
try:
|
| 824 |
+
self._run("""
|
| 825 |
+
MERGE (n:CourtCase {id:$id})
|
| 826 |
+
SET n.court_name=$court, n.state=$state,
|
| 827 |
+
n.pending_cases=$pending, n.disposed_cases=$disposed,
|
| 828 |
+
n.year=$year, n.source='NJDG',
|
| 829 |
+
n.dataset='njdg', n.scraped_at=$scraped_at
|
| 830 |
+
""", {
|
| 831 |
+
"id": node_id, "court": court,
|
| 832 |
+
"state": r.get("state",""),
|
| 833 |
+
"pending": int(r.get("pending_cases",0) or 0),
|
| 834 |
+
"disposed": int(r.get("disposed_cases",0) or 0),
|
| 835 |
+
"year": str(r.get("year","")),
|
| 836 |
+
"scraped_at":r.get("scraped_at",""),
|
| 837 |
+
})
|
| 838 |
+
count += 1
|
| 839 |
+
except Exception as e:
|
| 840 |
+
logger.warning(f"[Loader] CourtCase failed: {e}")
|
| 841 |
+
self.stats["court_cases"] = count
|
| 842 |
+
logger.success(f"[Loader] Loaded {count} court case records")
|
| 843 |
+
return count
|
| 844 |
+
|
| 845 |
+
def load_local_bodies(self, records: list) -> int:
|
| 846 |
+
"""LGD (Local Government Directory) → LocalBody nodes."""
|
| 847 |
+
count = 0
|
| 848 |
+
for r in records:
|
| 849 |
+
name = (r.get("name") or r.get("state_name") or "").strip()
|
| 850 |
+
if not name: continue
|
| 851 |
+
node_id = make_id(f"lgd_{r.get('lgd_code',name)}")
|
| 852 |
+
try:
|
| 853 |
+
self._run("""
|
| 854 |
+
MERGE (n:LocalBody {id:$id})
|
| 855 |
+
SET n.name=$name, n.lgd_code=$lgd_code,
|
| 856 |
+
n.entity_type=$entity_type, n.state=$state,
|
| 857 |
+
n.district=$district, n.source='LGD',
|
| 858 |
+
n.dataset='lgd', n.scraped_at=$scraped_at
|
| 859 |
+
""", {
|
| 860 |
+
"id": node_id, "name": name,
|
| 861 |
+
"lgd_code": str(r.get("lgd_code","")),
|
| 862 |
+
"entity_type":r.get("entity_type", r.get("type","")),
|
| 863 |
+
"state": r.get("state",""),
|
| 864 |
+
"district": r.get("district",""),
|
| 865 |
+
"scraped_at": r.get("scraped_at",""),
|
| 866 |
+
})
|
| 867 |
+
count += 1
|
| 868 |
+
except Exception as e:
|
| 869 |
+
logger.warning(f"[Loader] LocalBody failed: {e}")
|
| 870 |
+
self.stats["local_bodies"] = count
|
| 871 |
+
logger.success(f"[Loader] Loaded {count} local bodies")
|
| 872 |
+
return count
|
| 873 |
+
|
| 874 |
+
def load_crime_reports(self, records: list) -> int:
|
| 875 |
+
"""NCRB crime statistics — stored as metadata nodes for context."""
|
| 876 |
+
count = 0
|
| 877 |
+
for r in records:
|
| 878 |
+
state = (r.get("state") or "").strip()
|
| 879 |
+
year = str(r.get("year","")).strip()
|
| 880 |
+
if not state or not year: continue
|
| 881 |
+
node_id = make_id(f"ncrb_{state}_{year}")
|
| 882 |
+
try:
|
| 883 |
+
self._run("""
|
| 884 |
+
MERGE (n:CrimeReport {id:$id})
|
| 885 |
+
SET n.state=$state, n.year=$year,
|
| 886 |
+
n.crime_head=$crime_head,
|
| 887 |
+
n.cases_registered=$cases,
|
| 888 |
+
n.source='NCRB', n.dataset='ncrb',
|
| 889 |
+
n.scraped_at=$scraped_at
|
| 890 |
+
""", {
|
| 891 |
+
"id": node_id, "state": state, "year": year,
|
| 892 |
+
"crime_head": r.get("crime_head",""),
|
| 893 |
+
"cases": int(r.get("cases_registered",0) or 0),
|
| 894 |
+
"scraped_at": r.get("scraped_at",""),
|
| 895 |
+
})
|
| 896 |
+
count += 1
|
| 897 |
+
except Exception as e:
|
| 898 |
+
logger.warning(f"[Loader] CrimeReport failed: {e}")
|
| 899 |
+
self.stats["crime_reports"] = count
|
| 900 |
+
logger.success(f"[Loader] Loaded {count} NCRB crime reports")
|
| 901 |
+
return count
|
| 902 |
+
|
| 903 |
+
def load_wikidata_enrichments(self, records: list) -> int:
|
| 904 |
+
"""
|
| 905 |
+
Wikidata — enriches EXISTING Politician nodes; does NOT create new ones.
|
| 906 |
+
Uses MATCH not MERGE to avoid phantom nodes.
|
| 907 |
+
"""
|
| 908 |
+
count = 0
|
| 909 |
+
for r in records:
|
| 910 |
+
name = (r.get("name") or "").strip()
|
| 911 |
+
if not name: continue
|
| 912 |
+
try:
|
| 913 |
+
result = self._run("""
|
| 914 |
+
MATCH (p:Politician)
|
| 915 |
+
WHERE toLower(p.name) = toLower($name)
|
| 916 |
+
OR any(a IN coalesce(p.aliases,[]) WHERE toLower(a) = toLower($name))
|
| 917 |
+
SET p.wikidata_id = coalesce($wikidata_id, p.wikidata_id),
|
| 918 |
+
p.birth_year = coalesce($birth_year, p.birth_year),
|
| 919 |
+
p.gender = coalesce($gender, p.gender),
|
| 920 |
+
p.positions = coalesce($positions, p.positions),
|
| 921 |
+
p.wikidata_enriched = true
|
| 922 |
+
RETURN count(p) AS updated
|
| 923 |
+
""", {
|
| 924 |
+
"name": name,
|
| 925 |
+
"wikidata_id":r.get("wikidata_id",""),
|
| 926 |
+
"birth_year": str(r.get("birth_year","")),
|
| 927 |
+
"gender": r.get("gender",""),
|
| 928 |
+
"positions": str(r.get("positions",[])),
|
| 929 |
+
})
|
| 930 |
+
if result:
|
| 931 |
+
row = result.single()
|
| 932 |
+
if row and row["updated"] > 0:
|
| 933 |
+
count += 1
|
| 934 |
+
except Exception as e:
|
| 935 |
+
logger.warning(f"[Loader] Wikidata enrichment failed for {name}: {e}")
|
| 936 |
+
self.stats["wikidata_enrichments"] = count
|
| 937 |
+
logger.success(f"[Loader] Wikidata enriched {count} existing Politician nodes")
|
| 938 |
+
return count
|
| 939 |
+
|
| 940 |
+
def load_datagov_documents(self, records: list) -> int:
|
| 941 |
+
"""data.gov.in datasets — generic document nodes."""
|
| 942 |
+
count = 0
|
| 943 |
+
for r in records:
|
| 944 |
+
title = (r.get("title") or r.get("resource_title") or "").strip()
|
| 945 |
+
if not title: continue
|
| 946 |
+
node_id = make_id(f"datagov_{r.get('resource_id',title)}")
|
| 947 |
+
try:
|
| 948 |
+
self._run("""
|
| 949 |
+
MERGE (n:DataGovDocument {id:$id})
|
| 950 |
+
SET n.title=$title, n.dataset_name=$dataset_name,
|
| 951 |
+
n.resource_id=$resource_id,
|
| 952 |
+
n.ministry=$ministry,
|
| 953 |
+
n.source='data.gov.in', n.dataset='datagov',
|
| 954 |
+
n.scraped_at=$scraped_at
|
| 955 |
+
""", {
|
| 956 |
+
"id": node_id, "title": title,
|
| 957 |
+
"dataset_name":r.get("_dataset",""),
|
| 958 |
+
"resource_id": r.get("resource_id",""),
|
| 959 |
+
"ministry": r.get("ministry",""),
|
| 960 |
+
"scraped_at": r.get("scraped_at",""),
|
| 961 |
+
})
|
| 962 |
+
count += 1
|
| 963 |
+
except Exception as e:
|
| 964 |
+
logger.warning(f"[Loader] DataGovDocument failed: {e}")
|
| 965 |
+
self.stats["datagov_documents"] = count
|
| 966 |
+
logger.success(f"[Loader] Loaded {count} DataGov documents")
|
| 967 |
+
return count
|
| 968 |
+
|
| 969 |
def close(self):
|
| 970 |
if self.driver:
|
| 971 |
self.driver.close()
|
|
|
|
| 974 |
|
| 975 |
if __name__ == "__main__":
|
| 976 |
parser = argparse.ArgumentParser(description="BharatGraph Graph Loader")
|
| 977 |
+
parser.add_argument("--dry-run", action="store_true")
|
| 978 |
+
parser.add_argument("--file", type=str, default=None)
|
|
|
|
|
|
|
| 979 |
args = parser.parse_args()
|
| 980 |
|
| 981 |
print("=" * 55)
|
|
|
|
| 987 |
if args.file:
|
| 988 |
results = loader.load_from_pipeline_output(args.file)
|
| 989 |
else:
|
|
|
|
| 990 |
import glob
|
| 991 |
files = sorted(glob.glob("data/processed/pipeline_*.json"))
|
| 992 |
if files:
|
| 993 |
+
results = loader.load_from_pipeline_output(files[-1])
|
|
|
|
|
|
|
| 994 |
else:
|
| 995 |
+
print("\nNo pipeline output found. Run: python -m processing.pipeline")
|
|
|
|
| 996 |
results = {}
|
| 997 |
|
| 998 |
loader.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 999 |
s = loader.stats
|
| 1000 |
+
print(f"\n{'='*55}\nLOAD SUMMARY\n{'='*55}")
|
| 1001 |
+
print(f" Nodes created: {s['nodes_created']}")
|
| 1002 |
+
print(f" Nodes merged: {s['nodes_merged']}")
|
| 1003 |
+
print(f" Relationships: {s['rels_created']}")
|
| 1004 |
+
print(f" Errors: {s['errors']}")
|
| 1005 |
if args.dry_run:
|
| 1006 |
+
print(" (DRY RUN — nothing was written to Neo4j)")
|
| 1007 |
+
print("=" * 55)
|