move to pregenerated tokens

generic_ner.py

@@ -678,18 +678,18 @@ def remove_trailing_stopwords(entities):
 
 class MultitaskTokenClassificationPipeline(Pipeline):
 
-    # def _sanitize_parameters(self, **kwargs):
-    #     preprocess_kwargs = {}
-    #     print('kwargs', kwargs)
-    #     if "text" in kwargs:
-    #         preprocess_kwargs["text"] = kwargs["text"]
-    #     print("preprocess_kwargs", preprocess_kwargs)
-    #     self.label_map = self.model.config.label_map
-    #     self.id2label = {
-    #         task: {id_: label for label, id_ in labels.items()}
-    #         for task, labels in self.label_map.items()
-    #     }
-    #     return preprocess_kwargs, {}, {}
+    def _sanitize_parameters(self, **kwargs):
+        preprocess_kwargs = {}
+        print('kwargs', kwargs)
+        if "text" in kwargs:
+            preprocess_kwargs["text"] = kwargs["text"]
+        print("preprocess_kwargs", preprocess_kwargs)
+        self.label_map = self.model.config.label_map
+        self.id2label = {
+            task: {id_: label for label, id_ in labels.items()}
+            for task, labels in self.label_map.items()
+        }
+        return preprocess_kwargs, {}, {}
 
     def preprocess(self, inputs, **kwargs):