move to pregenerated tokens

generic_ner.py

@@ -678,18 +678,18 @@ def remove_trailing_stopwords(entities):
 
 class MultitaskTokenClassificationPipeline(Pipeline):
 
-    def _sanitize_parameters(self, **kwargs):
-        preprocess_kwargs = {}
-        print('kwargs', kwargs)
-        if "text" in kwargs:
-            preprocess_kwargs["text"] = kwargs["text"]
-        print("preprocess_kwargs", preprocess_kwargs)
-        self.label_map = self.model.config.label_map
-        self.id2label = {
-            task: {id_: label for label, id_ in labels.items()}
-            for task, labels in self.label_map.items()
-        }
-        return preprocess_kwargs, {}, {}
+    # def _sanitize_parameters(self, **kwargs):
+    #     preprocess_kwargs = {}
+    #     print('kwargs', kwargs)
+    #     if "text" in kwargs:
+    #         preprocess_kwargs["text"] = kwargs["text"]
+    #     print("preprocess_kwargs", preprocess_kwargs)
+    #     self.label_map = self.model.config.label_map
+    #     self.id2label = {
+    #         task: {id_: label for label, id_ in labels.items()}
+    #         for task, labels in self.label_map.items()
+    #     }
+    #     return preprocess_kwargs, {}, {}
 
     def preprocess(self, inputs, **kwargs):