Spaces:
Sleeping
Sleeping
File size: 2,980 Bytes
fbd78fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | """
Analytics Module — NEW feature not in original project
- Entity frequency distribution
- Co-occurrence analysis (which persons appear with which orgs)
- Confidence score statistics
- Entity deduplication with canonical form
"""
from collections import Counter, defaultdict
import re
def deduplicate_entities(entities: list[dict]) -> list[dict]:
"""
Remove duplicate mentions of the same entity.
'Barack Obama' and 'Obama' are treated as separate (conservative approach).
Case-insensitive deduplication within exact matches.
"""
seen = {}
result = []
for ent in entities:
key = (ent["word"].lower().strip(), ent["label"])
if key not in seen:
seen[key] = True
result.append(ent)
return result
def entity_frequency(entities: list[dict]) -> dict[str, Counter]:
"""
Returns per-label frequency counters.
e.g. {"Person": Counter({"Nafees Ahmad": 3, "Ali": 1}), ...}
"""
freq = defaultdict(Counter)
for ent in entities:
freq[ent["label"]][ent["word"]] += 1
return dict(freq)
def top_entities(entities: list[dict], top_n: int = 5) -> dict[str, list]:
"""Returns top N entities per label sorted by frequency."""
freq = entity_frequency(entities)
result = {}
for label, counter in freq.items():
result[label] = counter.most_common(top_n)
return result
def co_occurrence(entities: list[dict]) -> list[tuple]:
"""
Simple co-occurrence: find Person-Organization pairs that appear
in the same document. Useful for relationship extraction heuristic.
Returns list of (person, org) tuples.
"""
persons = [e["word"] for e in entities if e["label"] == "Person"]
orgs = [e["word"] for e in entities if e["label"] == "Organization"]
pairs = []
for person in set(persons):
for org in set(orgs):
pairs.append((person, org))
return pairs[:20] # limit output
def confidence_stats(entities: list[dict]) -> dict:
"""Return average, min, max confidence scores."""
if not entities:
return {"avg": 0, "min": 0, "max": 0, "total": 0}
scores = [e["score"] for e in entities]
return {
"avg": round(sum(scores) / len(scores), 1),
"min": round(min(scores), 1),
"max": round(max(scores), 1),
"total": len(entities),
}
def build_summary_table(entities: list[dict]) -> list[list]:
"""
Returns rows for Gradio DataFrame display.
Columns: Entity, Type, Confidence (%)
Sorted by confidence descending.
"""
sorted_ents = sorted(entities, key=lambda e: e["score"], reverse=True)
rows = [
[e["word"], e["label"], f'{e["score"]}%']
for e in sorted_ents
]
return rows
def label_counts(entities: list[dict]) -> dict[str, int]:
"""Count of entities per label."""
counts = defaultdict(int)
for e in entities:
counts[e["label"]] += 1
return dict(counts)
|