Spaces:
Running
Running
Commit ·
b3a84d6
1
Parent(s): 63af5ed
feat(processing): add AliasGraph -- alias-to-canonical-id lookup
Browse filesPersists and serves alias -> canonical_id mappings built by
EntityResolverV2. All lookups normalise to lowercase so
'RAHUL KUMAR', 'Rahul Kumar', and 'R. Kumar' all hit the same slot.
Supports bulk_add(), merge(), save()/load(), and stats().
Pure ASCII.
- processing/alias_graph.py +106 -0
processing/alias_graph.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
BharatGraph - Alias Graph
|
| 3 |
+
Maps every known name variant of an entity to its canonical ID.
|
| 4 |
+
Loaded at startup and used during all entity lookups so that
|
| 5 |
+
"RAHUL KUMAR", "Rahul Kumar", and "R. Kumar" all resolve to one node.
|
| 6 |
+
Pure ASCII.
|
| 7 |
+
"""
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from loguru import logger
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class AliasGraph:
|
| 14 |
+
"""
|
| 15 |
+
In-memory lookup: alias_name.lower() -> canonical_id
|
| 16 |
+
|
| 17 |
+
Built from EntityResolverV2.build_alias_graph() output and
|
| 18 |
+
persisted at data/processed/alias_graph.json.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
DEFAULT_PATH = "data/processed/alias_graph.json"
|
| 22 |
+
|
| 23 |
+
def __init__(self):
|
| 24 |
+
self._graph = {} # alias_lower -> canonical_id
|
| 25 |
+
|
| 26 |
+
def load(self, path: str = None) -> int:
|
| 27 |
+
"""Load alias graph from disk. Returns number of entries loaded."""
|
| 28 |
+
path = path or self.DEFAULT_PATH
|
| 29 |
+
if not os.path.exists(path):
|
| 30 |
+
logger.warning(
|
| 31 |
+
f"[AliasGraph] File not found: {path}. "
|
| 32 |
+
"Run pipeline first to build alias graph."
|
| 33 |
+
)
|
| 34 |
+
return 0
|
| 35 |
+
try:
|
| 36 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 37 |
+
self._graph = json.load(f)
|
| 38 |
+
logger.success(
|
| 39 |
+
f"[AliasGraph] Loaded {len(self._graph)} aliases from {path}"
|
| 40 |
+
)
|
| 41 |
+
return len(self._graph)
|
| 42 |
+
except Exception as e:
|
| 43 |
+
logger.error(f"[AliasGraph] Load failed: {e}")
|
| 44 |
+
return 0
|
| 45 |
+
|
| 46 |
+
def save(self, path: str = None):
|
| 47 |
+
"""Persist current alias graph to disk."""
|
| 48 |
+
path = path or self.DEFAULT_PATH
|
| 49 |
+
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
| 50 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 51 |
+
json.dump(self._graph, f, indent=2, ensure_ascii=False)
|
| 52 |
+
logger.success(
|
| 53 |
+
f"[AliasGraph] Saved {len(self._graph)} aliases to {path}"
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
def resolve(self, name: str) -> str:
|
| 57 |
+
"""
|
| 58 |
+
Resolve a name variant to its canonical ID.
|
| 59 |
+
Returns the canonical ID if found, else empty string.
|
| 60 |
+
"""
|
| 61 |
+
return self._graph.get(name.lower().strip(), "")
|
| 62 |
+
|
| 63 |
+
def add(self, alias: str, canonical_id: str):
|
| 64 |
+
"""Add or update a single alias -> canonical_id mapping."""
|
| 65 |
+
self._graph[alias.lower().strip()] = canonical_id
|
| 66 |
+
|
| 67 |
+
def merge(self, other: dict):
|
| 68 |
+
"""Merge another {alias: canonical_id} dict into this graph."""
|
| 69 |
+
normalised = {k.lower().strip(): v for k, v in other.items()}
|
| 70 |
+
self._graph.update(normalised)
|
| 71 |
+
logger.info(
|
| 72 |
+
f"[AliasGraph] Merged {len(other)} entries. "
|
| 73 |
+
f"Total: {len(self._graph)}"
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
def bulk_add(self, records: list, name_field: str,
|
| 77 |
+
canonical_id_field: str):
|
| 78 |
+
"""
|
| 79 |
+
Add aliases from a list of records.
|
| 80 |
+
Each record[name_field] becomes an alias for record[canonical_id_field].
|
| 81 |
+
"""
|
| 82 |
+
added = 0
|
| 83 |
+
for rec in records:
|
| 84 |
+
name = str(rec.get(name_field, "")).strip()
|
| 85 |
+
cid = str(rec.get(canonical_id_field, "")).strip()
|
| 86 |
+
if name and cid:
|
| 87 |
+
self._graph[name.lower()] = cid
|
| 88 |
+
added += 1
|
| 89 |
+
logger.info(f"[AliasGraph] Bulk-added {added} aliases")
|
| 90 |
+
|
| 91 |
+
def __len__(self) -> int:
|
| 92 |
+
return len(self._graph)
|
| 93 |
+
|
| 94 |
+
def __contains__(self, name: str) -> bool:
|
| 95 |
+
return name.lower().strip() in self._graph
|
| 96 |
+
|
| 97 |
+
def stats(self) -> dict:
|
| 98 |
+
"""Return basic statistics about the alias graph."""
|
| 99 |
+
canonical_ids = set(self._graph.values())
|
| 100 |
+
return {
|
| 101 |
+
"total_aliases": len(self._graph),
|
| 102 |
+
"unique_canonical_ids": len(canonical_ids),
|
| 103 |
+
"avg_aliases_per_id": round(
|
| 104 |
+
len(self._graph) / max(len(canonical_ids), 1), 2
|
| 105 |
+
),
|
| 106 |
+
}
|