Spaces:
Sleeping
Sleeping
File size: 2,951 Bytes
b7d0804 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
import re
from typing import List, Dict, Any
from app.graph.graph_quality import is_noisy_entity_name
STOP_ENTITIES = {
"The", "This", "That", "These", "Those", "It", "They", "We", "You",
"Page", "Chapter", "Figure", "Table", "Example", "Answer", "Question",
"Introduction", "Conclusion", "Summary", "Overview", "Paragraph",
"What", "Why", "When", "Where", "Who", "How", "Is", "Are", "IS"
}
def normalize_entity_name(name: str) -> str:
name = re.sub(r"\s+", " ", name or "").strip()
name = name.strip(".,;:()[]{}")
return name
def make_entity_id(name: str) -> str:
cleaned = name.lower()
cleaned = re.sub(r"[^a-z0-9]+", "_", cleaned)
cleaned = cleaned.strip("_")
return cleaned[:80] or "unknown_entity"
def classify_entity(name: str) -> str:
if re.fullmatch(r"[A-Z][A-Z0-9]{1,9}", name):
return "ACRONYM"
org_markers = [
"University", "Institute", "Corporation", "Corp", "Inc", "Ltd",
"Company", "OpenAI", "Microsoft", "Google", "Amazon"
]
if any(marker.lower() in name.lower() for marker in org_markers):
return "ORGANIZATION"
if any(char.isdigit() for char in name):
return "TECHNICAL_TERM"
if "-" in name or "/" in name:
return "TECHNICAL_TERM"
return "CONCEPT"
def is_valid_entity(name: str) -> bool:
if not name:
return False
if name in STOP_ENTITIES:
return False
if is_noisy_entity_name(name):
return False
if len(name) < 2:
return False
if len(name) > 90:
return False
return True
def extract_entities_from_text(text: str) -> List[Dict[str, Any]]:
if not text:
return []
candidates = []
# Acronyms like RAG, LLM, API, OCR, BM25
for match in re.finditer(r"\b[A-Z][A-Z0-9]{1,9}\b", text):
candidates.append(match.group(0))
# Capitalized technical phrases like Retrieval-Augmented Generation
capitalized_phrase_pattern = (
r"\b[A-Z][a-zA-Z0-9]*(?:[-/][A-Z]?[a-zA-Z0-9]+)?"
r"(?:\s+[A-Z][a-zA-Z0-9]*(?:[-/][A-Z]?[a-zA-Z0-9]+)?){0,5}\b"
)
for match in re.finditer(capitalized_phrase_pattern, text):
candidates.append(match.group(0))
cleaned_entities = []
seen = set()
for candidate in candidates:
name = normalize_entity_name(candidate)
if not is_valid_entity(name):
continue
entity_id = make_entity_id(name)
if entity_id in seen:
continue
seen.add(entity_id)
cleaned_entities.append(
{
"entity_id": entity_id,
"name": name,
"entity_type": classify_entity(name)
}
)
return cleaned_entities
def split_sentences(text: str) -> List[str]:
if not text:
return []
parts = re.split(r"(?<=[.!?])\s+", text)
return [part.strip() for part in parts if len(part.strip()) > 20]
|