Spaces:
Running
Running
Commit ·
6a0d2fb
1
Parent(s): aef2ed8
feat(phase-32): add 15 new tests for entity resolution edge cases
Browse filesNew test classes:
TestJaroWinklerIdentical: identical string = 1.0; prefix boost works
TestNormaliseEdgeCases: empty string, stacked honorifics (Late Shri),
Retd./Col., all 4 Pvt Ltd forms -> 1 canonical
TestEntityResolverScoring: wikidata_id exact match, GSTIN mismatch -> 0,
threshold boundary, source field preserved,
empty dataset -> []
TestAliasGraphEdgeCases: case-insensitive resolve, overwrite updates,
stats on empty graph
TestCanonicalIdEdgeCases: separator stability, politician vs company
produce different IDs, NGO with Darpan ID
Total test count: 44 (existing) + 15 (new) = 59 tests
- tests/test_entity_resolver_v2.py +141 -0
tests/test_entity_resolver_v2.py
CHANGED
|
@@ -220,3 +220,144 @@ class TestCanonicalId:
|
|
| 220 |
by_name = canonical_id_for_company(name="adani", state="gj")
|
| 221 |
assert by_cin != by_name # CIN and name-based IDs differ
|
| 222 |
assert len(by_cin) == 20
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
by_name = canonical_id_for_company(name="adani", state="gj")
|
| 221 |
assert by_cin != by_name # CIN and name-based IDs differ
|
| 222 |
assert len(by_cin) == 20
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
# ---- Phase 32 additions: 15 new tests for missing coverage ----
|
| 226 |
+
|
| 227 |
+
class TestJaroWinklerIdentical:
|
| 228 |
+
"""BUG: test_identical_strings was missing from the original test file."""
|
| 229 |
+
|
| 230 |
+
def test_identical_strings(self):
|
| 231 |
+
from processing.entity_resolver_v2 import jaro_winkler
|
| 232 |
+
assert jaro_winkler("modi", "modi") == 1.0
|
| 233 |
+
|
| 234 |
+
def test_prefix_boost(self):
|
| 235 |
+
"""Jaro-Winkler gives extra weight to common prefixes."""
|
| 236 |
+
from processing.entity_resolver_v2 import jaro_winkler
|
| 237 |
+
score_prefix = jaro_winkler("ramkumar", "ramesh")
|
| 238 |
+
score_no_pfx = jaro_winkler("kumar", "ramesh")
|
| 239 |
+
assert score_prefix > score_no_pfx
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
class TestNormaliseEdgeCases:
|
| 243 |
+
|
| 244 |
+
def test_empty_string(self):
|
| 245 |
+
from processing.entity_resolver_v2 import normalise_indian_name
|
| 246 |
+
result = normalise_indian_name("", "person")
|
| 247 |
+
assert result == ""
|
| 248 |
+
|
| 249 |
+
def test_stacked_honorifics(self):
|
| 250 |
+
"""Late Shri -> both honorifics stripped."""
|
| 251 |
+
from processing.entity_resolver_v2 import normalise_indian_name
|
| 252 |
+
result = normalise_indian_name("Late Shri Ram Kumar", "person")
|
| 253 |
+
assert "late" not in result
|
| 254 |
+
assert "shri" not in result
|
| 255 |
+
assert "ram kumar" in result
|
| 256 |
+
|
| 257 |
+
def test_retd_honorific(self):
|
| 258 |
+
from processing.entity_resolver_v2 import normalise_indian_name
|
| 259 |
+
result = normalise_indian_name("Retd. Col. Priya Singh", "person")
|
| 260 |
+
assert "retd" not in result
|
| 261 |
+
assert "col" not in result
|
| 262 |
+
|
| 263 |
+
def test_pvt_ltd_canonical(self):
|
| 264 |
+
from processing.entity_resolver_v2 import normalise_indian_name
|
| 265 |
+
forms = [
|
| 266 |
+
"Sample Pvt. Ltd.",
|
| 267 |
+
"Sample Private Limited",
|
| 268 |
+
"Sample Pvt Ltd",
|
| 269 |
+
"Sample P. Ltd.",
|
| 270 |
+
]
|
| 271 |
+
results = [normalise_indian_name(f, "company") for f in forms]
|
| 272 |
+
assert len(set(results)) == 1, f"Expected 1 canonical form, got: {set(results)}"
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
class TestEntityResolverScoring:
|
| 276 |
+
|
| 277 |
+
def test_wikidata_id_exact_match(self):
|
| 278 |
+
from processing.entity_resolver_v2 import EntityResolverV2
|
| 279 |
+
r = EntityResolverV2()
|
| 280 |
+
rec1 = {"name": "Narendra Modi", "wikidata_id": "Q658025"}
|
| 281 |
+
rec2 = {"name": "N. Modi", "wikidata_id": "Q658025"}
|
| 282 |
+
assert r.combined_score("Narendra Modi", "N. Modi", rec1, rec2) == 1.0
|
| 283 |
+
|
| 284 |
+
def test_gstin_mismatch_returns_zero(self):
|
| 285 |
+
from processing.entity_resolver_v2 import EntityResolverV2
|
| 286 |
+
r = EntityResolverV2()
|
| 287 |
+
rec1 = {"name": "Company X", "gstin": "27AAACM0629R1ZU"}
|
| 288 |
+
rec2 = {"name": "Company X", "gstin": "29AAACM0629R1ZU"}
|
| 289 |
+
assert r.combined_score("Company X", "Company X", rec1, rec2) == 0.0
|
| 290 |
+
|
| 291 |
+
def test_threshold_boundary(self):
|
| 292 |
+
from processing.entity_resolver_v2 import EntityResolverV2
|
| 293 |
+
r = EntityResolverV2(threshold=0.90)
|
| 294 |
+
# Perfect match
|
| 295 |
+
assert r.is_same_entity("RAHUL KUMAR", "Rahul Kumar")
|
| 296 |
+
# Clearly different
|
| 297 |
+
assert not r.is_same_entity("Alpha Corp", "Beta Industries", kind="company")
|
| 298 |
+
|
| 299 |
+
def test_resolve_dataset_preserves_source_field(self):
|
| 300 |
+
from processing.entity_resolver_v2 import EntityResolverV2
|
| 301 |
+
r = EntityResolverV2(threshold=0.72)
|
| 302 |
+
records = [
|
| 303 |
+
{"name": "RAM KUMAR", "_source": "myneta", "state": "UP"},
|
| 304 |
+
{"name": "Ram Kumar", "_source": "mca", "state": "UP"},
|
| 305 |
+
]
|
| 306 |
+
resolved = r.resolve_dataset(records, "name")
|
| 307 |
+
assert len(resolved) == 1
|
| 308 |
+
assert resolved[0]["_source"] == "myneta" # first record wins
|
| 309 |
+
|
| 310 |
+
def test_resolve_empty_dataset(self):
|
| 311 |
+
from processing.entity_resolver_v2 import EntityResolverV2
|
| 312 |
+
r = EntityResolverV2()
|
| 313 |
+
assert r.resolve_dataset([], "name") == []
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
class TestAliasGraphEdgeCases:
|
| 317 |
+
|
| 318 |
+
def test_resolve_case_insensitive(self):
|
| 319 |
+
from processing.alias_graph import AliasGraph
|
| 320 |
+
ag = AliasGraph()
|
| 321 |
+
ag.add("Narendra Modi", "pol_001")
|
| 322 |
+
assert ag.resolve("NARENDRA MODI") == "pol_001"
|
| 323 |
+
assert ag.resolve("narendra modi") == "pol_001"
|
| 324 |
+
|
| 325 |
+
def test_overwrite_updates_canonical(self):
|
| 326 |
+
from processing.alias_graph import AliasGraph
|
| 327 |
+
ag = AliasGraph()
|
| 328 |
+
ag.add("Test Name", "old_id")
|
| 329 |
+
ag.add("Test Name", "new_id")
|
| 330 |
+
assert ag.resolve("test name") == "new_id"
|
| 331 |
+
|
| 332 |
+
def test_stats_empty_graph(self):
|
| 333 |
+
from processing.alias_graph import AliasGraph
|
| 334 |
+
ag = AliasGraph()
|
| 335 |
+
stats = ag.stats()
|
| 336 |
+
assert stats["total_aliases"] == 0
|
| 337 |
+
assert stats["unique_canonical_ids"] == 0
|
| 338 |
+
assert stats["avg_aliases_per_id"] == 0
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
class TestCanonicalIdEdgeCases:
|
| 342 |
+
|
| 343 |
+
def test_separator_in_name(self):
|
| 344 |
+
from processing.canonical_id import canonical_id
|
| 345 |
+
# Pipe is the internal separator -- should still produce stable ID
|
| 346 |
+
id1 = canonical_id("test|name", "state")
|
| 347 |
+
id2 = canonical_id("test|name", "state")
|
| 348 |
+
assert id1 == id2
|
| 349 |
+
assert len(id1) == 20
|
| 350 |
+
|
| 351 |
+
def test_politician_vs_company_different_ids(self):
|
| 352 |
+
from processing.canonical_id import canonical_id_for_politician, canonical_id_for_company
|
| 353 |
+
pol = canonical_id_for_politician("Ram Kumar", "UP")
|
| 354 |
+
com = canonical_id_for_company(name="Ram Kumar", state="UP")
|
| 355 |
+
# Same name+state but different function prefix -> different IDs
|
| 356 |
+
assert pol != com
|
| 357 |
+
|
| 358 |
+
def test_ngo_with_darpan_id(self):
|
| 359 |
+
from processing.canonical_id import canonical_id_for_ngo
|
| 360 |
+
id1 = canonical_id_for_ngo(darpan_id="DL/2021/0012345")
|
| 361 |
+
id2 = canonical_id_for_ngo(darpan_id="DL/2021/0012345")
|
| 362 |
+
assert id1 == id2
|
| 363 |
+
assert len(id1) == 20
|