Spaces:
Sleeping
Sleeping
Commit Β·
06270b4
1
Parent(s): 9f87a81
fix(encoding): strip non-ASCII chars from pipeline.py and loader.py
Browse filesBoth files contained UTF-8 em dashes (U+2014) and arrows (U+2192) in
comments which cause UnicodeDecodeError on GitHub Actions runners that
read files without explicit encoding= argument.
Replaced all non-ASCII punctuation with ASCII equivalents:
U+2014 (em dash) -> --
U+2192 (arrow) -> ->
All other logic is unchanged.
- graph/loader.py +16 -16
- processing/pipeline.py +2 -2
graph/loader.py
CHANGED
|
@@ -36,7 +36,7 @@ class GraphLoader:
|
|
| 36 |
"errors": 0,
|
| 37 |
}
|
| 38 |
if dry_run:
|
| 39 |
-
logger.info("[Loader] DRY RUN mode
|
| 40 |
elif driver is None:
|
| 41 |
self._connect()
|
| 42 |
|
|
@@ -104,8 +104,8 @@ class GraphLoader:
|
|
| 104 |
"""
|
| 105 |
Execute a single Cypher query.
|
| 106 |
BUG-17 FIX: now accepts both calling styles:
|
| 107 |
-
- old style: self._run(query, {"key": val})
|
| 108 |
-
- new style: self._run(query, key=val, ...)
|
| 109 |
Both are merged so existing callers and the 8 new loaders both work.
|
| 110 |
"""
|
| 111 |
if self.dry_run:
|
|
@@ -121,7 +121,7 @@ class GraphLoader:
|
|
| 121 |
self.stats["errors"] += 1
|
| 122 |
return None
|
| 123 |
|
| 124 |
-
#
|
| 125 |
|
| 126 |
def load_politicians(self, records: list) -> int:
|
| 127 |
count = 0
|
|
@@ -399,7 +399,7 @@ class GraphLoader:
|
|
| 399 |
count += 1
|
| 400 |
self.stats["rels_created"] += 1
|
| 401 |
except Exception as e:
|
| 402 |
-
logger.warning(f"[Loader] DIRECTOR_OF link {pol_name}
|
| 403 |
|
| 404 |
logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
|
| 405 |
return count
|
|
@@ -430,7 +430,7 @@ class GraphLoader:
|
|
| 430 |
if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
|
| 431 |
if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
|
| 432 |
if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
|
| 433 |
-
# BUG-2 FIX: 7 previously missing loaders
|
| 434 |
# silently dropped because load_from_pipeline_output never called them.
|
| 435 |
if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
|
| 436 |
if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
|
|
@@ -443,7 +443,7 @@ class GraphLoader:
|
|
| 443 |
logger.success(f"[Loader] Load complete. Stats: {self.stats}")
|
| 444 |
return {**results, "stats": self.stats}
|
| 445 |
|
| 446 |
-
#
|
| 447 |
|
| 448 |
def load_regulatory_orders(self, records: list) -> int:
|
| 449 |
count = 0
|
|
@@ -739,10 +739,10 @@ class GraphLoader:
|
|
| 739 |
logger.success(f"[Loader] Loaded {count} CVC circulars")
|
| 740 |
return count
|
| 741 |
|
| 742 |
-
#
|
| 743 |
|
| 744 |
def load_icij_entities(self, records: list) -> int:
|
| 745 |
-
"""ICIJ Offshore Leaks entities
|
| 746 |
count = 0
|
| 747 |
for r in records:
|
| 748 |
name = (r.get("name") or r.get("entity_name") or "").strip()
|
|
@@ -778,7 +778,7 @@ class GraphLoader:
|
|
| 778 |
return count
|
| 779 |
|
| 780 |
def load_sanctioned_entities(self, records: list) -> int:
|
| 781 |
-
"""OpenSanctions
|
| 782 |
count = 0
|
| 783 |
for r in records:
|
| 784 |
name = (r.get("name") or r.get("caption") or "").strip()
|
|
@@ -814,7 +814,7 @@ class GraphLoader:
|
|
| 814 |
return count
|
| 815 |
|
| 816 |
def load_court_cases(self, records: list) -> int:
|
| 817 |
-
"""NJDG court pendency stats
|
| 818 |
count = 0
|
| 819 |
for r in records:
|
| 820 |
court = (r.get("court_name") or r.get("state") or "").strip()
|
|
@@ -843,7 +843,7 @@ class GraphLoader:
|
|
| 843 |
return count
|
| 844 |
|
| 845 |
def load_local_bodies(self, records: list) -> int:
|
| 846 |
-
"""LGD (Local Government Directory)
|
| 847 |
count = 0
|
| 848 |
for r in records:
|
| 849 |
name = (r.get("name") or r.get("state_name") or "").strip()
|
|
@@ -872,7 +872,7 @@ class GraphLoader:
|
|
| 872 |
return count
|
| 873 |
|
| 874 |
def load_crime_reports(self, records: list) -> int:
|
| 875 |
-
"""NCRB crime statistics
|
| 876 |
count = 0
|
| 877 |
for r in records:
|
| 878 |
state = (r.get("state") or "").strip()
|
|
@@ -902,7 +902,7 @@ class GraphLoader:
|
|
| 902 |
|
| 903 |
def load_wikidata_enrichments(self, records: list) -> int:
|
| 904 |
"""
|
| 905 |
-
Wikidata
|
| 906 |
Uses MATCH not MERGE to avoid phantom nodes.
|
| 907 |
"""
|
| 908 |
count = 0
|
|
@@ -938,7 +938,7 @@ class GraphLoader:
|
|
| 938 |
return count
|
| 939 |
|
| 940 |
def load_datagov_documents(self, records: list) -> int:
|
| 941 |
-
"""data.gov.in datasets
|
| 942 |
count = 0
|
| 943 |
for r in records:
|
| 944 |
title = (r.get("title") or r.get("resource_title") or "").strip()
|
|
@@ -1003,5 +1003,5 @@ if __name__ == "__main__":
|
|
| 1003 |
print(f" Relationships: {s['rels_created']}")
|
| 1004 |
print(f" Errors: {s['errors']}")
|
| 1005 |
if args.dry_run:
|
| 1006 |
-
print(" (DRY RUN
|
| 1007 |
print("=" * 55)
|
|
|
|
| 36 |
"errors": 0,
|
| 37 |
}
|
| 38 |
if dry_run:
|
| 39 |
+
logger.info("[Loader] DRY RUN mode -- no data written to Neo4j")
|
| 40 |
elif driver is None:
|
| 41 |
self._connect()
|
| 42 |
|
|
|
|
| 104 |
"""
|
| 105 |
Execute a single Cypher query.
|
| 106 |
BUG-17 FIX: now accepts both calling styles:
|
| 107 |
+
- old style: self._run(query, {"key": val}) <- positional dict
|
| 108 |
+
- new style: self._run(query, key=val, ...) <- keyword args
|
| 109 |
Both are merged so existing callers and the 8 new loaders both work.
|
| 110 |
"""
|
| 111 |
if self.dry_run:
|
|
|
|
| 121 |
self.stats["errors"] += 1
|
| 122 |
return None
|
| 123 |
|
| 124 |
+
# ?? Node loaders ??????????????????????????????????????
|
| 125 |
|
| 126 |
def load_politicians(self, records: list) -> int:
|
| 127 |
count = 0
|
|
|
|
| 399 |
count += 1
|
| 400 |
self.stats["rels_created"] += 1
|
| 401 |
except Exception as e:
|
| 402 |
+
logger.warning(f"[Loader] DIRECTOR_OF link {pol_name}->{co_name} failed: {e}")
|
| 403 |
|
| 404 |
logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
|
| 405 |
return count
|
|
|
|
| 430 |
if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
|
| 431 |
if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
|
| 432 |
if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
|
| 433 |
+
# BUG-2 FIX: 7 previously missing loaders -- these datasets were scraped but
|
| 434 |
# silently dropped because load_from_pipeline_output never called them.
|
| 435 |
if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
|
| 436 |
if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
|
|
|
|
| 443 |
logger.success(f"[Loader] Load complete. Stats: {self.stats}")
|
| 444 |
return {**results, "stats": self.stats}
|
| 445 |
|
| 446 |
+
# ?? Phase 28 loaders (8 datasets) ????????????????????????????????????????
|
| 447 |
|
| 448 |
def load_regulatory_orders(self, records: list) -> int:
|
| 449 |
count = 0
|
|
|
|
| 739 |
logger.success(f"[Loader] Loaded {count} CVC circulars")
|
| 740 |
return count
|
| 741 |
|
| 742 |
+
# ?? BUG-2 FIX: 7 NEW loaders -- were scraped but never loaded ?????????????
|
| 743 |
|
| 744 |
def load_icij_entities(self, records: list) -> int:
|
| 745 |
+
"""ICIJ Offshore Leaks entities -> ICIJEntity nodes."""
|
| 746 |
count = 0
|
| 747 |
for r in records:
|
| 748 |
name = (r.get("name") or r.get("entity_name") or "").strip()
|
|
|
|
| 778 |
return count
|
| 779 |
|
| 780 |
def load_sanctioned_entities(self, records: list) -> int:
|
| 781 |
+
"""OpenSanctions -> SanctionedEntity nodes."""
|
| 782 |
count = 0
|
| 783 |
for r in records:
|
| 784 |
name = (r.get("name") or r.get("caption") or "").strip()
|
|
|
|
| 814 |
return count
|
| 815 |
|
| 816 |
def load_court_cases(self, records: list) -> int:
|
| 817 |
+
"""NJDG court pendency stats -> CourtCase nodes."""
|
| 818 |
count = 0
|
| 819 |
for r in records:
|
| 820 |
court = (r.get("court_name") or r.get("state") or "").strip()
|
|
|
|
| 843 |
return count
|
| 844 |
|
| 845 |
def load_local_bodies(self, records: list) -> int:
|
| 846 |
+
"""LGD (Local Government Directory) -> LocalBody nodes."""
|
| 847 |
count = 0
|
| 848 |
for r in records:
|
| 849 |
name = (r.get("name") or r.get("state_name") or "").strip()
|
|
|
|
| 872 |
return count
|
| 873 |
|
| 874 |
def load_crime_reports(self, records: list) -> int:
|
| 875 |
+
"""NCRB crime statistics -- stored as metadata nodes for context."""
|
| 876 |
count = 0
|
| 877 |
for r in records:
|
| 878 |
state = (r.get("state") or "").strip()
|
|
|
|
| 902 |
|
| 903 |
def load_wikidata_enrichments(self, records: list) -> int:
|
| 904 |
"""
|
| 905 |
+
Wikidata -- enriches EXISTING Politician nodes; does NOT create new ones.
|
| 906 |
Uses MATCH not MERGE to avoid phantom nodes.
|
| 907 |
"""
|
| 908 |
count = 0
|
|
|
|
| 938 |
return count
|
| 939 |
|
| 940 |
def load_datagov_documents(self, records: list) -> int:
|
| 941 |
+
"""data.gov.in datasets -- generic document nodes."""
|
| 942 |
count = 0
|
| 943 |
for r in records:
|
| 944 |
title = (r.get("title") or r.get("resource_title") or "").strip()
|
|
|
|
| 1003 |
print(f" Relationships: {s['rels_created']}")
|
| 1004 |
print(f" Errors: {s['errors']}")
|
| 1005 |
if args.dry_run:
|
| 1006 |
+
print(" (DRY RUN -- nothing was written to Neo4j)")
|
| 1007 |
print("=" * 55)
|
processing/pipeline.py
CHANGED
|
@@ -28,7 +28,7 @@ class BharatGraphPipeline:
|
|
| 28 |
os.makedirs("data/samples", exist_ok=True)
|
| 29 |
logger.info("[Pipeline] Initialized")
|
| 30 |
|
| 31 |
-
#
|
| 32 |
|
| 33 |
def run_datagov(self) -> list:
|
| 34 |
try:
|
|
@@ -383,7 +383,7 @@ class BharatGraphPipeline:
|
|
| 383 |
"politician_co_links":len(links),
|
| 384 |
}
|
| 385 |
|
| 386 |
-
logger.info(f"[Pipeline] Done in {duration}s
|
| 387 |
f"{summary['total_raw_records']} total records")
|
| 388 |
|
| 389 |
results = {
|
|
|
|
| 28 |
os.makedirs("data/samples", exist_ok=True)
|
| 29 |
logger.info("[Pipeline] Initialized")
|
| 30 |
|
| 31 |
+
# ?? Scrapers ?????????????????????????????????????????????????????????????
|
| 32 |
|
| 33 |
def run_datagov(self) -> list:
|
| 34 |
try:
|
|
|
|
| 383 |
"politician_co_links":len(links),
|
| 384 |
}
|
| 385 |
|
| 386 |
+
logger.info(f"[Pipeline] Done in {duration}s -- "
|
| 387 |
f"{summary['total_raw_records']} total records")
|
| 388 |
|
| 389 |
results = {
|