File size: 4,370 Bytes
05ef53c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 | import spacy
import re
import unicodedata
class TextProcessor:
"""
Utilize spaCy NLP model to process text
"""
def __init__(self):
self.nlp = spacy.load("en_core_web_sm")
self.document_filter = self.build_exclusion_pattern([
'chapter',
'topic'
])
# Greek char -> Latin term mappings
self.GREEK_MAP = {
# Upper case
'Α': 'alpha', 'Β': 'beta', 'Γ': 'gamma', '∆': 'delta', 'Ε': 'epsilon',
'Ζ': 'zeta', 'Η': 'eta', 'Θ': 'theta', 'Ι': 'iota', 'Κ': 'kappa',
'Λ': 'lambda', 'Μ': 'mu', 'Ν': 'nu', 'Ξ': 'xi', 'Ο': 'omicron',
'Π': 'pi', 'Ρ': 'rho', 'Σ': 'sigma', 'Τ': 'tau', 'Υ': 'upsilon',
'Φ': 'phi', 'Χ': 'chi', 'Ψ': 'psi', 'Ω': 'omega',
# Lower case
'α': 'alpha', 'β': 'beta', 'γ': 'gamma', 'δ': 'delta', 'ε': 'epsilon',
'ζ': 'zeta', 'η': 'eta', 'θ': 'theta', 'ι': 'iota', 'κ': 'kappa',
'λ': 'lambda', 'μ': 'mu', 'ν': 'nu', 'ξ': 'xi', 'ο': 'omicron',
'π': 'pi', 'ρ': 'rho', 'σ': 'sigma', 'ς': 'sigma', 'τ': 'tau',
'υ': 'upsilon', 'φ': 'phi', 'χ': 'chi', 'ψ': 'psi', 'ω': 'omega'
}
self.GREEK_PATTERN = re.compile("|".join(map(re.escape, self.GREEK_MAP.keys())))
def split_sentences(self, text: str):
"""
Function to split a paragraph into valid sentences
"""
doc = self.nlp(text)
ret = []
for sent in doc.sents:
notrait_text = sent.text.strip()
if notrait_text and not self.document_filter.search(notrait_text):
ret.append(self.clean_text(notrait_text))
return ret
def lemmatize(self, text: str):
"""
Function to reduce words inside a string into their base dictionary form
"""
doc = self.nlp(text)
return " ".join([token.lemma_ for token in doc])
def clean_text(self, text):
"""
Function to eliminate section strings
"""
# Matches: I. OVERVIEW, II. STRUCTURE, III. ...
section_pattern = r"\b[IVXLCDM]+\.\s+[A-Z][A-Z\s]+\b"
return re.sub(section_pattern, "", text)
def build_exclusion_pattern(self, keywords):
"""
Builder function for words filtering
"""
# escape keywords to avoid regex issues
escaped = [re.escape(k) for k in keywords]
pattern = r"\b(" + "|".join(escaped) + r")\b"
return re.compile(pattern, re.IGNORECASE)
def simplify_phrase(self, text: str):
"""
Function to simplify a phrase into its core structure using dependency parsing
"""
# Parse the phrase
doc = self.nlp(text)
# Retrieve the root action, fallback to original phrase if failed
root = next((t for t in doc if t.dep_ == "ROOT"), None)
if not root:
return text
# Retrieve the root action's dependencies
parts = [root]
for child in root.children:
if child.dep_ in {"prep", "agent", "prt"}:
parts.append(child)
parts = sorted(parts, key=lambda x: x.i)
return " ".join(t.text for t in parts)
def normalize_text(self, text: str) -> str:
"""
Canonical text normalization:
- Unicode normalize
- Expand Greek character into corresponding Latin term
- Remove spaces around hypens
- Replace hypen separators with spaces
- Normalize separators
- Remove unwanted symbols
- Collapse spaces
- Lowercase + strip
"""
text = unicodedata.normalize("NFKC", text)
# Expand Greek
for k, v in self.GREEK_MAP.items():
text = text.replace(k, v)
# Normalize spaced hyphens
text = re.sub(r"\s*-\s*", "-", text)
# Replace hyphen between letters with space
text = re.sub(r"(?<=[a-zA-Z])-(?=[a-zA-Z])", " ", text)
# Normalize other separators
text = re.sub(r"[,+/]", " ", text)
# Remove unwanted chars
text = re.sub(r"[()\[\]\|]'", "", text)
# Collapse spaces
text = re.sub(r"\s+", " ", text)
return text.strip().lower()
|