Update spaCy pipeline

README.md

@@ -9,7 +9,7 @@ language:
 | --- | --- |
 | **Name** | `it_trf_reflex_nrp` |
 | **Version** | `0.0.0` |
-| **spaCy** | `>=3.8.3,<3.9.0` |
+| **spaCy** | `>=3.8.7,<3.9.0` |
 | **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1a74c01c415516deee963c0565d4b7a041db7ff1d9b24c499e8618275595068
-size 443821766
+oid sha256:198bd1e70ebfb9931065cb51fbf76dd01f7568664d7c3f2bc59ad26eb520ad3a
+size 443822163

config.cfg

@@ -16,7 +16,7 @@ before_creation = null
 after_creation = null
 after_pipeline_creation = null
 batch_size = 1000
-tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+tokenizer = {"@tokenizers":"customize_tokenizer"}
 vectors = {"@vectors":"spacy.Vectors.v1"}
 
 [components]
@@ -90,7 +90,7 @@ set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotati
 
 [components.ner_transformer.model]
 @architectures = "spacy-transformers.TransformerModel.v3"
-name = "dbmdz/bert-base-german-cased"
+name = "dbmdz/bert-base-italian-xxl-cased"
 mixed_precision = false
 
 [components.ner_transformer.model.get_spans]
@@ -228,21 +228,24 @@ eps = 0.00000001
 learn_rate = 0.001
 
 [training.score_weights]
-ents_f = 0.2
+ents_f = 0.22
 ents_p = 0.0
 ents_r = 0.0
 ents_per_type = null
-pos_acc = 0.1
-morph_acc = 0.1
+pos_acc = 0.0
+morph_acc = 0.11
 morph_per_feat = null
-tag_acc = 0.2
-dep_uas = 0.1
-dep_las = 0.1
+tag_acc = 0.22
+tag_micro_p = null
+tag_micro_r = null
+tag_micro_f = null
+dep_uas = 0.11
+dep_las = 0.11
 dep_las_per_type = null
 sents_p = null
 sents_r = null
 sents_f = 0.0
-lemma_acc = 0.2
+lemma_acc = 0.22
 
 [pretraining]
 

it_trf_reflex_nrp-0.0.0-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18413e743fc44bba9879b8909bdc91aa27985762aa118e5eec6069c0e7f299e4
-size 825614552
+oid sha256:2652e97f0f00f72f8a8f3ee70503ebaabeb8b4e07f8b925b24c1204ec63d8804
+size 828275955

meta.json

@@ -7,8 +7,8 @@
   "email":"",
   "url":"",
   "license":"",
-  "spacy_version":">=3.8.3,<3.9.0",
-  "spacy_git_version":"be0fa81",
+  "spacy_version":">=3.8.7,<3.9.0",
+  "spacy_git_version":"4b65aa7",
   "vectors":{
     "width":0,
     "vectors":0,
@@ -846,7 +846,7 @@
 
   ],
   "requirements":[
-    "spacy-transformers>=1.3.8,<1.4.0",
-    "spacy>=3.8.3,<3.9.0"
+    "spacy-transformers>=1.3.9,<1.4.0",
+    "spacy>=3.8.7,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de1ee4820116e2b59c685bd2f1db214678f8dd13cd3a95fc64ce0cdd0ea935b5
+oid sha256:5032987473186a0f62555d61538ee0907d892791e1a74a01d3119edd84723f28
 size 219690

ner/moves

@@ -1 +1 @@
-��movesٴ{"0":{},"1":{"ORG":4516,"LOC":3499,"PER":2987},"2":{"ORG":4516,"LOC":3499,"PER":2987},"3":{"ORG":4516,"LOC":3499,"PER":2987},"4":{"ORG":4516,"LOC":3499,"PER":2987,"":1},"5":{"":1}}�cfg��neg_key�
+��movesٴ{"0":{},"1":{"ORG":5497,"LOC":4378,"PER":3735},"2":{"ORG":5497,"LOC":4378,"PER":3735},"3":{"ORG":5497,"LOC":4378,"PER":3735},"4":{"ORG":5497,"LOC":4378,"PER":3735,"":1},"5":{"":1}}�cfg��neg_key�

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f04bb5d044576ea0166a033d8c34ca45ddd944370799409ad15da3e8d873536
-size 440759212
+oid sha256:b49959037f8c416a86998262af8eb9e38fdf1e1c0ffb41cd29d65c62475106b8
+size 443822170

use_custom_tokenizer.py

@@ -24,17 +24,26 @@ EXTENDED_LETTER_RANGE = "A-Za-zäöüÄÖÜàòèéìù"
 DATE = r"[0-3][1-9]\.[0-1][1-9]\.[1-2][0-9]{3}"
 TOP_LEVEL_DOMAINS = "ch|at|de|com|edu|org|gov|net|fr|uk|be|es|pl|it|eu|nl|ba|cz|dk|al|ad|bg|by|fi|gr|ie|li|lu|no|pt|ro|rs|ru|se|si|sk"
 
+DOT_BEFORE_WORD = [
+    rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{{i}}})(?!(({TOP_LEVEL_DOMAINS})\b))"
+    for i in range(3, 30)
+]
+
+
 DOT_AFTER_WORD = [
-    rf"(?<!www\.)(?<=([a-zA-ZäöüÄÖÜ]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
+    rf"(?<!www\.)(?<=([{EXTENDED_LETTER_RANGE}]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
     for i in range(3, 30)
 ]
 
+
 DOT_AFTER_DATE = rf"(?<=({DATE}))\."
 
 infix_res = [
-    r"[\(\[\]\)]",
+    r"[\(\[\]\)\,\»\>\«\<]",  # brackets, commas and ' that are without space
     r"(?<=\.--)\.",  # DOT after .--
-    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    r"(?<=\,)\'",
+    *DOT_BEFORE_WORD,  # DOT before word
+    # rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",
     r"'\.\.",  # e.g., 'Tscheicha'.. -> "Tscheicha" "'..", then split ".." as suffix
     *DOT_AFTER_WORD,  # when there is no space after the dot
     r"[A-Z](?=\. )",  # DOT after capital letter
@@ -45,6 +54,11 @@ LETTER_DOUBLE_ENDING_DOT_VAR_LENGTH = [  # DOT after letter, e.g., A.G., or u.s.
     rf"(?<=([{EXTENDED_LETTER_RANGE}]\.){{{i}}})\." for i in range(1, 30)
 ]
 
+prefix_res = [
+    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    *DOT_BEFORE_WORD,  # DOT before word
+]
+
 suffix_res = [
     r"(?<=\d)[\.]",  # DOT after number
     r"(?<=[\.])[\]\)]",  # Closing brackets with DOT before
@@ -63,10 +77,12 @@ NUMBER_SIGN_NUMBER_FIXED = r"(?<=[0-9])[+\*^](?=[0-9])"
 
 
 # Given a nlp object, return a custom tokenizer that splits on special cases and with unwanted tokenization removed
-def custom_tokenizer(nlp):
+def custom_tokenizer(nlp) -> Tokenizer:
     nlp.tokenizer = Tokenizer(nlp.vocab)
 
-    prefix_regex = compile_prefix_regex(nlp.Defaults.prefixes)
+    prefixes = nlp.Defaults.prefixes
+    prefixes += prefix_res
+    prefix_regex = compile_prefix_regex(prefixes)
     nlp.tokenizer.prefix_search = prefix_regex.search
 
     # We use the default infixes and remove some cases that lead to unwanted tokenization.