Update spaCy pipeline

README.md

@@ -26,8 +26,8 @@ model-index:
 | **Name** | `en_setec_mk_tv` |
 | **Version** | `0.0.0` |
 | **spaCy** | `>=3.7.5,<3.8.0` |
-| **Default Pipeline** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component` |
-| **Components** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component` |
+| **Default Pipeline** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component`, `feature_aggregator_component` |
+| **Components** | `tok2vec`, `ner`, `count_extraction_component`, `normalizer_component`, `feature_aggregator_component` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |

config.cfg

@@ -10,7 +10,7 @@ seed = 0
 
 [nlp]
 lang = "en"
-pipeline = ["tok2vec","ner","count_extraction_component","normalizer_component"]
+pipeline = ["tok2vec","ner","count_extraction_component","normalizer_component","feature_aggregator_component"]
 batch_size = 1000
 disabled = []
 before_creation = null
@@ -25,6 +25,41 @@ vectors = {"@vectors":"spacy.Vectors.v1"}
 factory = "count_extraction_component"
 label = "CONNECTION"
 
+[components.feature_aggregator_component]
+factory = "feature_aggregator_component"
+
+[components.feature_aggregator_component.config]
+
+[components.feature_aggregator_component.config.AUDIO_FEATURE]
+method = "first"
+
+[components.feature_aggregator_component.config.COLOR]
+method = "join"
+
+[components.feature_aggregator_component.config.INCH]
+method = "first"
+
+[components.feature_aggregator_component.config.MOUNTING_FEATURE]
+method = "join"
+
+[components.feature_aggregator_component.config.OS]
+method = "first"
+
+[components.feature_aggregator_component.config.REFRESH_RATE]
+method = "first"
+
+[components.feature_aggregator_component.config.RESOLUTION]
+method = "first"
+
+[components.feature_aggregator_component.config.SOFTWARE_FEATURE]
+method = "expand"
+
+[components.feature_aggregator_component.config.VIDEO_FEATURE]
+method = "expand"
+
+[components.feature_aggregator_component.config.WIRELESS_FEATURE]
+method = "expand"
+
 [components.ner]
 factory = "ner"
 incorrect_spans_key = null
@@ -48,7 +83,7 @@ upstream = "*"
 
 [components.normalizer_component]
 factory = "normalizer_component"
-norm_file = null
+norms = null
 
 [components.tok2vec]
 factory = "tok2vec"

en_setec_mk_tv-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa2fe9279a4fe64e62863259d1b3a8412df014f391706cd345415ea9776b3d27
-size 5707735
+oid sha256:95caa99fc72a38765852ae3e92072e5c28bf4357f166a16a2cfff6969b5c03e9
+size 5709056

feature_aggregator_component.py

@@ -0,0 +1,101 @@
+from spacy.tokens import Doc, Span, Token
+from spacy.language import Language
+
+default_feature_aggregation_config = {
+    'AUDIO_FEATURE': {
+        'method': 'first',
+    },
+    'COLOR': {
+        'method': 'join',
+    },
+    'INCH': {
+        'method': 'first',
+    },
+    'MOUNTING_FEATURE': {
+        'method': 'join',
+    },
+    'OS': {
+        'method': 'first',
+    },
+    'REFRESH_RATE': {
+        'method': 'first',
+    },
+    'RESOLUTION': {
+        'method': 'first',
+    },
+    'SOFTWARE_FEATURE': {
+        'method': 'expand',
+    },
+    'VIDEO_FEATURE': {
+        'method': 'expand',
+    },
+    'WIRELESS_FEATURE': {
+        'method': 'expand',
+    },
+}
+
+@Language.factory("feature_aggregator_component")
+class FeatureAggregatorComponent(object):
+    def __init__(self, nlp, name, config=default_feature_aggregation_config):
+        self.config = config
+        Doc.set_extension("raw_features", getter=self.raw_features, force=True)
+        Doc.set_extension("features", getter=self.features, force=True)
+        Doc.set_extension("add_to_dataframe", method=self.add_to_dataframe, force=True)
+        Doc.set_extension("feature_aggregation_config", getter=self.get_feature_aggregation_config, setter=self.set_feature_aggregation_config, force=True)
+        
+    def __call__(self, doc):
+        return doc
+    
+    def get_feature_aggregation_config(self, doc):
+        return self.config
+    
+    def set_feature_aggregation_config(self, doc, config):
+        self.config = config
+
+    def raw_features(self, doc):
+        features = {}
+        for ent in doc.ents:
+            if ent._.count is None:
+                if not ent.label_ in features:
+                    features[ent.label_] = set()
+                features[ent.label_].add(ent._.text)
+            else:
+                # If it has a count we put it in a separate column an accumulate the counts
+                if not ent._.text in features:
+                    features[ent._.text] = 0
+                features[ent._.text] += ent._.count
+        return features
+
+    def features(self, doc):
+        features = {}
+        for name, values in self.raw_features(doc).items():
+            if not name in self.config:
+                features[name] = values
+                continue
+            
+            if not 'method' in self.config[name]:
+                features[name] = values
+                continue
+
+            method = self.config[name]["method"]
+            if method == 'first':
+                if len(values) != 0:
+                    features[name] = values.pop()
+                else:
+                    features[name] = float('nan')
+            elif method == 'join':
+                features[name] = ','.join(list(values))
+            elif method == 'expand':
+                for value in values:
+                    features[value] = 1
+            else:
+                print(f"unknown feature aggregation method: {method}, skipping...")
+                features[name] = values
+        return features
+    
+    def add_to_dataframe(self, doc, df):
+        features = self.features(doc)
+        for name, feature in features.items():
+            features[name] = [feature]
+        df = pd.concat([df, pd.DataFrame(features)])
+        return df

meta.json

@@ -37,13 +37,15 @@
     "tok2vec",
     "ner",
     "count_extraction_component",
-    "normalizer_component"
+    "normalizer_component",
+    "feature_aggregator_component"
   ],
   "components":[
     "tok2vec",
     "ner",
     "count_extraction_component",
-    "normalizer_component"
+    "normalizer_component",
+    "feature_aggregator_component"
   ],
   "disabled":[
 

normalizer_component.py

@@ -50,19 +50,19 @@ default_normalization_table = {
     "1280x720": ["HD"],
     "640x480": ["SD"],
     "Wifi": ["Wifi", "Wi-Fi", "Wifi built in", "built in Wifi", "WiFi integrated"],
-    "BLUETOOTH": ["BLUETOOTH", "Blutooth"],
+    "Blutooth": ["BLUETOOTH"],
 }
 
 
 @Language.factory("normalizer_component")
 class NormalizerComponent(object):
-    def __init__(self, nlp, name, norm_file=None):
-        # if norm_file is None:
-        self.norm_table = default_normalization_table
-        # elif isinstance(norm_file, object):
-        #     self.norm_table = norm_file
-        # else:
-        #     self.norm_table = json.load(open(norm_file))
+    def __init__(self, nlp, name, norms=None):
+        if norms is None:
+            self.norm_table = default_normalization_table
+        elif isinstance(norms, str):
+            self.norm_table = json.load(open(norms))
+        else:
+            self.norm_table = norms
 
         self.matcher = PhraseMatcher(nlp.vocab, attr="LOWER")
         self.nlp = nlp