| | import json |
| |
|
| | from spacy.language import Language |
| | from spacy.matcher import PhraseMatcher |
| |
|
| | |
| | default_normalization_table = { |
| | "Dolby Atmos": ["Dolby Atmos", "Dolby Audio Atmos", "Dolby Atmos Audio"], |
| | "Ethernet": [ |
| | "Ethernet LAN", |
| | "Ethernet port RJ-45", |
| | "Ethernet RJ-45", |
| | "Ethernet RJ45", |
| | "Ethernet-LAN RJ-45", |
| | "LAN RJ45", |
| | "Ethernet R45", |
| | ], |
| | "CI+ Slot": [ |
| | "CI+ Card Slot", |
| | "Common Interface Plus (CI+)", |
| | "Common Interface Plus", |
| | "Card Slot CI +", |
| | ], |
| | "Scart": ["SCART", "Scart Input"], |
| | "Component In": [ |
| | "Component In", |
| | "Component in(YPbPr)", |
| | "Component Input", |
| | "Component (Y/Pb/Pr)", |
| | "Component In (Y/Pb/Pr)", |
| | ], |
| | "USB 2.0": ["USB2.0"], |
| | "Digital Audio": [ |
| | "Digital Audio Out", |
| | "Digital Audio Output", |
| | "Digital Audio Output(Coaxial and Optic)", |
| | ], |
| | "Composite In": ["Composite", "AV Composite In"], |
| | "3.5mm Headphone jack": ["3.5mm Headphone jack", "Headphone 3.5mm jack"], |
| | "Optical Audio Out": ["Optical Audio Out", "Optical Out"], |
| | "Android": ["ANDROID"], |
| | "Android 7.1": ["Android Nougat"], |
| | "Google TV": ["GoogleTV", "Google LED TV", "Google miniLED TV", "Google OLED TV"], |
| | "VIDAA U4": ["VIDAA U4.0"], |
| | "Android TV": ["Android TV", "AndroidTV", "Android"], |
| | "Titan OS": ["TITAN OS"], |
| | "7680x4320": ["8K"], |
| | "3840x2160": ["4K", "4K UltraHD", "4K Ultra HD", "UltraHD", "Ultra HD"], |
| | "1920x1080": ["FullHD", "Full HD"], |
| | "1366x768": ["HD Ready", "HDReady"], |
| | "1280x720": ["HD"], |
| | "640x480": ["SD"], |
| | "Wifi": ["Wifi", "Wi-Fi", "Wifi built in", "built in Wifi", "WiFi integrated"], |
| | "Blutooth": ["BLUETOOTH"], |
| | } |
| |
|
| |
|
| | @Language.factory("normalizer_component") |
| | class NormalizerComponent(object): |
| | def __init__(self, nlp, name, norms=None): |
| | if norms is None: |
| | self.norm_table = default_normalization_table |
| | elif isinstance(norms, str): |
| | self.norm_table = json.load(open(norms)) |
| | else: |
| | self.norm_table = norms |
| |
|
| | self.matcher = PhraseMatcher(nlp.vocab, attr="LOWER") |
| | self.nlp = nlp |
| |
|
| | for name, patterns in self.norm_table.items(): |
| | self.matcher.add(name, [nlp.make_doc(pattern) for pattern in patterns]) |
| |
|
| | def __call__(self, doc): |
| | for ent in doc.ents: |
| | for match_id, start, end in self.matcher(self.nlp.make_doc(ent._.text)): |
| | match_id_string = self.nlp.vocab.strings[match_id] |
| | ent._.text = match_id_string.strip() |
| | return doc |
| |
|
