Update spaCy pipeline

README.md

@@ -26,8 +26,8 @@ model-index:
 | **Name** | `it_trf_nrp` |
 | **Version** | `0.0.0` |
 | **spaCy** | `>=3.8.3,<3.9.0` |
-| **Default Pipeline** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
-| **Components** | `ner_transformer`, `ner`, `merge_entities`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
+| **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
 | **Sources** | n/a |
 | **License** | n/a |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc821393399de287ef8756d0ee3ea86ced3e509d0f5196a8d9d8ca32a54507b5
+oid sha256:382b251bd81e6c65fe4033674bc76b90eee93409d6c567aafbfc11d94c2e22cc
 size 443821706

config.cfg

@@ -10,7 +10,7 @@ seed = 17
 
 [nlp]
 lang = "it"
-pipeline = ["ner_transformer","ner","merge_entities","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
+pipeline = ["ner_transformer","ner","base_transformer","morphologizer","tagger","parser","trainable_lemmatizer"]
 batch_size = 512
 disabled = []
 before_creation = null
@@ -43,9 +43,6 @@ use_fast = true
 
 [components.base_transformer.model.transformer_config]
 
-[components.merge_entities]
-factory = "merge_entities"
-
 [components.morphologizer]
 factory = "morphologizer"
 extend = false

it_trf_nrp-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e36485c33f32539c299614cba011ff61388adb7ce4464f9adb8e268f4c36824d
-size 825599590
+oid sha256:e1c331f6965c59e36deb2bb546991791a0055e2abd1c39a00bda81d5590bdacd
+size 825600865

meta.json

@@ -827,7 +827,6 @@
   "pipeline":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",
@@ -837,7 +836,6 @@
   "components":[
     "ner_transformer",
     "ner",
-    "merge_entities",
     "base_transformer",
     "morphologizer",
     "tagger",

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8241c52299b144357c0ee8000b1246d52fad6788100ba00a24392e0a2819f39
+oid sha256:82e6707a6d7df1d48c0ade9bf95d437a23daf1a125a1f5457d957a29a007be3a
 size 440759145

use_custom_tokenizer.py

@@ -1,13 +1,12 @@
 from spacy.util import registry
-
-from commercial_registry_ner.spacy.custom_tokenizer.custom_tokenizer import (
-    custom_tokenizer,
-)
-
+from spacy.tokenizer import Tokenizer
+import pathlib
 
 @registry.tokenizers("customize_tokenizer")
 def make_customize_tokenizer():
     def customize_tokenizer(nlp):
-        return custom_tokenizer(nlp)
+        tokenizer = Tokenizer(nlp.vocab)
+        script_dir = pathlib.Path(__file__).parent.resolve()
+        return tokenizer.from_disk(script_dir / "tokenizer")
 
     return customize_tokenizer