abinazebinoy commited on
Commit
06270b4
Β·
1 Parent(s): 9f87a81

fix(encoding): strip non-ASCII chars from pipeline.py and loader.py

Browse files

Both files contained UTF-8 em dashes (U+2014) and arrows (U+2192) in
comments which cause UnicodeDecodeError on GitHub Actions runners that
read files without explicit encoding= argument.

Replaced all non-ASCII punctuation with ASCII equivalents:
U+2014 (em dash) -> --
U+2192 (arrow) -> ->

All other logic is unchanged.

Files changed (2) hide show
  1. graph/loader.py +16 -16
  2. processing/pipeline.py +2 -2
graph/loader.py CHANGED
@@ -36,7 +36,7 @@ class GraphLoader:
36
  "errors": 0,
37
  }
38
  if dry_run:
39
- logger.info("[Loader] DRY RUN mode β€” no data written to Neo4j")
40
  elif driver is None:
41
  self._connect()
42
 
@@ -104,8 +104,8 @@ class GraphLoader:
104
  """
105
  Execute a single Cypher query.
106
  BUG-17 FIX: now accepts both calling styles:
107
- - old style: self._run(query, {"key": val}) ← positional dict
108
- - new style: self._run(query, key=val, ...) ← keyword args
109
  Both are merged so existing callers and the 8 new loaders both work.
110
  """
111
  if self.dry_run:
@@ -121,7 +121,7 @@ class GraphLoader:
121
  self.stats["errors"] += 1
122
  return None
123
 
124
- # ── Node loaders ──────────────────────────────────────
125
 
126
  def load_politicians(self, records: list) -> int:
127
  count = 0
@@ -399,7 +399,7 @@ class GraphLoader:
399
  count += 1
400
  self.stats["rels_created"] += 1
401
  except Exception as e:
402
- logger.warning(f"[Loader] DIRECTOR_OF link {pol_name}β†’{co_name} failed: {e}")
403
 
404
  logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
405
  return count
@@ -430,7 +430,7 @@ class GraphLoader:
430
  if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
431
  if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
432
  if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
433
- # BUG-2 FIX: 7 previously missing loaders β€” these datasets were scraped but
434
  # silently dropped because load_from_pipeline_output never called them.
435
  if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
436
  if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
@@ -443,7 +443,7 @@ class GraphLoader:
443
  logger.success(f"[Loader] Load complete. Stats: {self.stats}")
444
  return {**results, "stats": self.stats}
445
 
446
- # ── Phase 28 loaders (8 datasets) ────────────────────────────────────────
447
 
448
  def load_regulatory_orders(self, records: list) -> int:
449
  count = 0
@@ -739,10 +739,10 @@ class GraphLoader:
739
  logger.success(f"[Loader] Loaded {count} CVC circulars")
740
  return count
741
 
742
- # ── BUG-2 FIX: 7 NEW loaders β€” were scraped but never loaded ─────────────
743
 
744
  def load_icij_entities(self, records: list) -> int:
745
- """ICIJ Offshore Leaks entities β†’ ICIJEntity nodes."""
746
  count = 0
747
  for r in records:
748
  name = (r.get("name") or r.get("entity_name") or "").strip()
@@ -778,7 +778,7 @@ class GraphLoader:
778
  return count
779
 
780
  def load_sanctioned_entities(self, records: list) -> int:
781
- """OpenSanctions β†’ SanctionedEntity nodes."""
782
  count = 0
783
  for r in records:
784
  name = (r.get("name") or r.get("caption") or "").strip()
@@ -814,7 +814,7 @@ class GraphLoader:
814
  return count
815
 
816
  def load_court_cases(self, records: list) -> int:
817
- """NJDG court pendency stats β†’ CourtCase nodes."""
818
  count = 0
819
  for r in records:
820
  court = (r.get("court_name") or r.get("state") or "").strip()
@@ -843,7 +843,7 @@ class GraphLoader:
843
  return count
844
 
845
  def load_local_bodies(self, records: list) -> int:
846
- """LGD (Local Government Directory) β†’ LocalBody nodes."""
847
  count = 0
848
  for r in records:
849
  name = (r.get("name") or r.get("state_name") or "").strip()
@@ -872,7 +872,7 @@ class GraphLoader:
872
  return count
873
 
874
  def load_crime_reports(self, records: list) -> int:
875
- """NCRB crime statistics β€” stored as metadata nodes for context."""
876
  count = 0
877
  for r in records:
878
  state = (r.get("state") or "").strip()
@@ -902,7 +902,7 @@ class GraphLoader:
902
 
903
  def load_wikidata_enrichments(self, records: list) -> int:
904
  """
905
- Wikidata β€” enriches EXISTING Politician nodes; does NOT create new ones.
906
  Uses MATCH not MERGE to avoid phantom nodes.
907
  """
908
  count = 0
@@ -938,7 +938,7 @@ class GraphLoader:
938
  return count
939
 
940
  def load_datagov_documents(self, records: list) -> int:
941
- """data.gov.in datasets β€” generic document nodes."""
942
  count = 0
943
  for r in records:
944
  title = (r.get("title") or r.get("resource_title") or "").strip()
@@ -1003,5 +1003,5 @@ if __name__ == "__main__":
1003
  print(f" Relationships: {s['rels_created']}")
1004
  print(f" Errors: {s['errors']}")
1005
  if args.dry_run:
1006
- print(" (DRY RUN β€” nothing was written to Neo4j)")
1007
  print("=" * 55)
 
36
  "errors": 0,
37
  }
38
  if dry_run:
39
+ logger.info("[Loader] DRY RUN mode -- no data written to Neo4j")
40
  elif driver is None:
41
  self._connect()
42
 
 
104
  """
105
  Execute a single Cypher query.
106
  BUG-17 FIX: now accepts both calling styles:
107
+ - old style: self._run(query, {"key": val}) <- positional dict
108
+ - new style: self._run(query, key=val, ...) <- keyword args
109
  Both are merged so existing callers and the 8 new loaders both work.
110
  """
111
  if self.dry_run:
 
121
  self.stats["errors"] += 1
122
  return None
123
 
124
+ # ?? Node loaders ??????????????????????????????????????
125
 
126
  def load_politicians(self, records: list) -> int:
127
  count = 0
 
399
  count += 1
400
  self.stats["rels_created"] += 1
401
  except Exception as e:
402
+ logger.warning(f"[Loader] DIRECTOR_OF link {pol_name}->{co_name} failed: {e}")
403
 
404
  logger.info(f"[Loader] DIRECTOR_OF links created/updated: {count}")
405
  return count
 
430
  if raw.get("cppp"): results["tenders"] = self.load_tenders(raw["cppp"])
431
  if raw.get("loksabha"): results["parliament_questions"]= self.load_parliament_questions(raw["loksabha"])
432
  if raw.get("cvc"): results["vigilance_circulars"] = self.load_vigilance_circulars(raw["cvc"])
433
+ # BUG-2 FIX: 7 previously missing loaders -- these datasets were scraped but
434
  # silently dropped because load_from_pipeline_output never called them.
435
  if raw.get("icij"): results["icij_entities"] = self.load_icij_entities(raw["icij"])
436
  if raw.get("opensanctions"):results["sanctioned_entities"] = self.load_sanctioned_entities(raw["opensanctions"])
 
443
  logger.success(f"[Loader] Load complete. Stats: {self.stats}")
444
  return {**results, "stats": self.stats}
445
 
446
+ # ?? Phase 28 loaders (8 datasets) ????????????????????????????????????????
447
 
448
  def load_regulatory_orders(self, records: list) -> int:
449
  count = 0
 
739
  logger.success(f"[Loader] Loaded {count} CVC circulars")
740
  return count
741
 
742
+ # ?? BUG-2 FIX: 7 NEW loaders -- were scraped but never loaded ?????????????
743
 
744
  def load_icij_entities(self, records: list) -> int:
745
+ """ICIJ Offshore Leaks entities -> ICIJEntity nodes."""
746
  count = 0
747
  for r in records:
748
  name = (r.get("name") or r.get("entity_name") or "").strip()
 
778
  return count
779
 
780
  def load_sanctioned_entities(self, records: list) -> int:
781
+ """OpenSanctions -> SanctionedEntity nodes."""
782
  count = 0
783
  for r in records:
784
  name = (r.get("name") or r.get("caption") or "").strip()
 
814
  return count
815
 
816
  def load_court_cases(self, records: list) -> int:
817
+ """NJDG court pendency stats -> CourtCase nodes."""
818
  count = 0
819
  for r in records:
820
  court = (r.get("court_name") or r.get("state") or "").strip()
 
843
  return count
844
 
845
  def load_local_bodies(self, records: list) -> int:
846
+ """LGD (Local Government Directory) -> LocalBody nodes."""
847
  count = 0
848
  for r in records:
849
  name = (r.get("name") or r.get("state_name") or "").strip()
 
872
  return count
873
 
874
  def load_crime_reports(self, records: list) -> int:
875
+ """NCRB crime statistics -- stored as metadata nodes for context."""
876
  count = 0
877
  for r in records:
878
  state = (r.get("state") or "").strip()
 
902
 
903
  def load_wikidata_enrichments(self, records: list) -> int:
904
  """
905
+ Wikidata -- enriches EXISTING Politician nodes; does NOT create new ones.
906
  Uses MATCH not MERGE to avoid phantom nodes.
907
  """
908
  count = 0
 
938
  return count
939
 
940
  def load_datagov_documents(self, records: list) -> int:
941
+ """data.gov.in datasets -- generic document nodes."""
942
  count = 0
943
  for r in records:
944
  title = (r.get("title") or r.get("resource_title") or "").strip()
 
1003
  print(f" Relationships: {s['rels_created']}")
1004
  print(f" Errors: {s['errors']}")
1005
  if args.dry_run:
1006
+ print(" (DRY RUN -- nothing was written to Neo4j)")
1007
  print("=" * 55)
processing/pipeline.py CHANGED
@@ -28,7 +28,7 @@ class BharatGraphPipeline:
28
  os.makedirs("data/samples", exist_ok=True)
29
  logger.info("[Pipeline] Initialized")
30
 
31
- # ── Scrapers ─────────────────────────────────────────────────────────────
32
 
33
  def run_datagov(self) -> list:
34
  try:
@@ -383,7 +383,7 @@ class BharatGraphPipeline:
383
  "politician_co_links":len(links),
384
  }
385
 
386
- logger.info(f"[Pipeline] Done in {duration}s β€” "
387
  f"{summary['total_raw_records']} total records")
388
 
389
  results = {
 
28
  os.makedirs("data/samples", exist_ok=True)
29
  logger.info("[Pipeline] Initialized")
30
 
31
+ # ?? Scrapers ?????????????????????????????????????????????????????????????
32
 
33
  def run_datagov(self) -> list:
34
  try:
 
383
  "politician_co_links":len(links),
384
  }
385
 
386
+ logger.info(f"[Pipeline] Done in {duration}s -- "
387
  f"{summary['total_raw_records']} total records")
388
 
389
  results = {