Update spaCy pipeline

README.md

@@ -9,7 +9,7 @@ language:
 | --- | --- |
 | **Name** | `de_trf_reflex_nrp` |
 | **Version** | `0.0.0` |
-| **spaCy** | `>=3.8.3,<3.9.0` |
+| **spaCy** | `>=3.8.7,<3.9.0` |
 | **Default Pipeline** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Components** | `ner_transformer`, `ner`, `base_transformer`, `morphologizer`, `tagger`, `parser`, `trainable_lemmatizer` |
 | **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |

base_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01c4a6205b2ab5026ba2064eb50e10d668530bc92f42ed9b3d5b6adf718f1163
-size 440759205
+oid sha256:fe87b1b20d663c6b3a4639ac363c010d54c8dd148277205284de0dfefbac246c
+size 440759602

config.cfg

@@ -16,7 +16,7 @@ before_creation = null
 after_creation = null
 after_pipeline_creation = null
 batch_size = 1000
-tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+tokenizer = {"@tokenizers":"customize_tokenizer"}
 vectors = {"@vectors":"spacy.Vectors.v1"}
 
 [components]
@@ -228,21 +228,24 @@ eps = 0.00000001
 learn_rate = 0.001
 
 [training.score_weights]
-ents_f = 0.2
+ents_f = 0.22
 ents_p = 0.0
 ents_r = 0.0
 ents_per_type = null
-pos_acc = 0.1
-morph_acc = 0.1
+pos_acc = 0.0
+morph_acc = 0.11
 morph_per_feat = null
-tag_acc = 0.2
-dep_uas = 0.1
-dep_las = 0.1
+tag_acc = 0.22
+tag_micro_p = null
+tag_micro_r = null
+tag_micro_f = null
+dep_uas = 0.11
+dep_las = 0.11
 dep_las_per_type = null
 sents_p = null
 sents_r = null
 sents_f = 0.0
-lemma_acc = 0.2
+lemma_acc = 0.22
 
 [pretraining]
 

de_trf_reflex_nrp-0.0.0-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4fd4fd6042c3d9fa691f5931d278afc60d6fd85c280b03dfb561d5c8d33c265
-size 865728325
+oid sha256:8847b573ed146775d3d3258db3ae3c39f23920d3c5e1117d0db18d774380eb0f
+size 865731526

meta.json

@@ -7,8 +7,8 @@
   "email":"",
   "url":"",
   "license":"",
-  "spacy_version":">=3.8.3,<3.9.0",
-  "spacy_git_version":"be0fa81",
+  "spacy_version":">=3.8.7,<3.9.0",
+  "spacy_git_version":"4b65aa7",
   "vectors":{
     "width":0,
     "vectors":0,
@@ -1849,7 +1849,7 @@
 
   ],
   "requirements":[
-    "spacy-transformers>=1.3.8,<1.4.0",
-    "spacy>=3.8.3,<3.9.0"
+    "spacy-transformers>=1.3.9,<1.4.0",
+    "spacy>=3.8.7,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0693e3662125cda25380a809a5937f1c78ce427c643bf0eb5973c78a99b8389
+oid sha256:969c0055d7b1e700d7ad7b138e74aba275f3a5adb3db1c5692047e4d60ff58e9
 size 219690

ner/moves

@@ -1 +1 @@
-��movesٴ{"0":{},"1":{"ORG":5064,"LOC":4932,"PER":4078},"2":{"ORG":5064,"LOC":4932,"PER":4078},"3":{"ORG":5064,"LOC":4932,"PER":4078},"4":{"ORG":5064,"LOC":4932,"PER":4078,"":1},"5":{"":1}}�cfg��neg_key�
+��movesٴ{"0":{},"1":{"LOC":5568,"ORG":5550,"PER":4662},"2":{"LOC":5568,"ORG":5550,"PER":4662},"3":{"LOC":5568,"ORG":5550,"PER":4662},"4":{"LOC":5568,"ORG":5550,"PER":4662,"":1},"5":{"":1}}�cfg��neg_key�

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a29a195d4656bc285e8dba14c78e1020f8c2e22db156d0c8ea83e7cb1a914bef
-size 440759212
+oid sha256:7c94683b8cce69d51530a2a91de1a4dee720b1dc60e2abbdb702f9d0d08db7fd
+size 440759609

use_custom_tokenizer.py

@@ -24,17 +24,26 @@ EXTENDED_LETTER_RANGE = "A-Za-zäöüÄÖÜàòèéìù"
 DATE = r"[0-3][1-9]\.[0-1][1-9]\.[1-2][0-9]{3}"
 TOP_LEVEL_DOMAINS = "ch|at|de|com|edu|org|gov|net|fr|uk|be|es|pl|it|eu|nl|ba|cz|dk|al|ad|bg|by|fi|gr|ie|li|lu|no|pt|ro|rs|ru|se|si|sk"
 
+DOT_BEFORE_WORD = [
+    rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{{i}}})(?!(({TOP_LEVEL_DOMAINS})\b))"
+    for i in range(3, 30)
+]
+
+
 DOT_AFTER_WORD = [
-    rf"(?<!www\.)(?<=([a-zA-ZäöüÄÖÜ]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
+    rf"(?<!www\.)(?<=([{EXTENDED_LETTER_RANGE}]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
     for i in range(3, 30)
 ]
 
+
 DOT_AFTER_DATE = rf"(?<=({DATE}))\."
 
 infix_res = [
-    r"[\(\[\]\)]",
+    r"[\(\[\]\)\,\»\>\«\<]",  # brackets, commas and ' that are without space
     r"(?<=\.--)\.",  # DOT after .--
-    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    r"(?<=\,)\'",
+    *DOT_BEFORE_WORD,  # DOT before word
+    # rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",
     r"'\.\.",  # e.g., 'Tscheicha'.. -> "Tscheicha" "'..", then split ".." as suffix
     *DOT_AFTER_WORD,  # when there is no space after the dot
     r"[A-Z](?=\. )",  # DOT after capital letter
@@ -45,6 +54,11 @@ LETTER_DOUBLE_ENDING_DOT_VAR_LENGTH = [  # DOT after letter, e.g., A.G., or u.s.
     rf"(?<=([{EXTENDED_LETTER_RANGE}]\.){{{i}}})\." for i in range(1, 30)
 ]
 
+prefix_res = [
+    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    *DOT_BEFORE_WORD,  # DOT before word
+]
+
 suffix_res = [
     r"(?<=\d)[\.]",  # DOT after number
     r"(?<=[\.])[\]\)]",  # Closing brackets with DOT before
@@ -63,10 +77,12 @@ NUMBER_SIGN_NUMBER_FIXED = r"(?<=[0-9])[+\*^](?=[0-9])"
 
 
 # Given a nlp object, return a custom tokenizer that splits on special cases and with unwanted tokenization removed
-def custom_tokenizer(nlp):
+def custom_tokenizer(nlp) -> Tokenizer:
     nlp.tokenizer = Tokenizer(nlp.vocab)
 
-    prefix_regex = compile_prefix_regex(nlp.Defaults.prefixes)
+    prefixes = nlp.Defaults.prefixes
+    prefixes += prefix_res
+    prefix_regex = compile_prefix_regex(prefixes)
     nlp.tokenizer.prefix_search = prefix_regex.search
 
     # We use the default infixes and remove some cases that lead to unwanted tokenization.