Update spaCy pipeline

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 de_trf_ner_base_reflex_nrp-0.0.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
 ner/model filter=lfs diff=lfs merge=lfs -text
 ner_transformer/model filter=lfs diff=lfs merge=lfs -text
+de_trf_ner_base_reflex_nrp-1.0.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -13,18 +13,18 @@ model-index:
     metrics:
     - name: NER Precision
       type: precision
-      value: 0.9710610932
+      value: 0.9773609314
     - name: NER Recall
       type: recall
-      value: 0.971842317
+      value: 0.9798962387
     - name: NER F Score
       type: f_score
-      value: 0.971451548
+      value: 0.978626943
 ---
 | Feature | Description |
 | --- | --- |
 | **Name** | `de_trf_ner_base_reflex_nrp` |
-| **Version** | `0.0.0` |
+| **Version** | `1.0.0` |
 | **spaCy** | `>=3.8.3,<3.9.0` |
 | **Default Pipeline** | `ner_transformer`, `ner` |
 | **Components** | `ner_transformer`, `ner` |
@@ -49,8 +49,8 @@ model-index:
 
 | Type | Score |
 | --- | --- |
-| `ENTS_F` | 97.15 |
-| `ENTS_P` | 97.11 |
-| `ENTS_R` | 97.18 |
-| `NER_TRANSFORMER_LOSS` | 4331.69 |
-| `NER_LOSS` | 33002.60 |
+| `ENTS_F` | 97.86 |
+| `ENTS_P` | 97.74 |
+| `ENTS_R` | 97.99 |
+| `NER_TRANSFORMER_LOSS` | 3997.89 |
+| `NER_LOSS` | 20826.48 |

config.cfg

@@ -1,6 +1,6 @@
 [paths]
-train = "./data/raw/training/02-2025/de/train/combined.spacy"
-dev = "./data/raw/training/02-2025/de/dev/combined.spacy"
+train = "./dataset/corpus/de/07_2025/train/combined.spacy"
+dev = "./dataset/corpus/de/07_2025/dev/combined.spacy"
 vectors = null
 init_tok2vec = null
 

de_trf_ner_base_reflex_nrp-1.0.0-py3-none-any.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57317d04dacd709a9e5dac248f648c94952f41b178e33a14dcda9bfeeb0a7fff
+size 408982425

meta.json

@@ -1,7 +1,7 @@
 {
   "lang":"de",
   "name":"trf_ner_base_reflex_nrp",
-  "version":"0.0.0",
+  "version":"1.0.0",
   "description":"",
   "author":"",
   "email":"",
@@ -37,31 +37,31 @@
 
   ],
   "performance":{
-    "ents_f":0.971451548,
-    "ents_p":0.9710610932,
-    "ents_r":0.971842317,
+    "ents_f":0.978626943,
+    "ents_p":0.9773609314,
+    "ents_r":0.9798962387,
     "ents_per_type":{
       "ORG":{
-        "p":0.9395973154,
-        "r":0.9523809524,
-        "f":0.9459459459
+        "p":0.9484848485,
+        "r":0.9456193353,
+        "f":0.9470499244
       },
       "LOC":{
-        "p":0.981186686,
-        "r":0.9755395683,
-        "f":0.9783549784
+        "p":0.9919168591,
+        "r":0.9930635838,
+        "f":0.9924898902
       },
       "PER":{
-        "p":0.9803921569,
-        "r":0.9842519685,
-        "f":0.9823182711
+        "p":0.9685714286,
+        "r":0.9797687861,
+        "f":0.974137931
       }
     },
-    "ner_transformer_loss":43.3168994993,
-    "ner_loss":330.0260414947
+    "ner_transformer_loss":39.9788878204,
+    "ner_loss":208.264759525
   },
   "requirements":[
-    "spacy-transformers>=1.3.8,<1.4.0",
+    "spacy-transformers>=1.3.9,<1.4.0",
     "spacy>=3.8.3,<3.9.0"
   ]
 }

ner/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0693e3662125cda25380a809a5937f1c78ce427c643bf0eb5973c78a99b8389
+oid sha256:969c0055d7b1e700d7ad7b138e74aba275f3a5adb3db1c5692047e4d60ff58e9
 size 219690

ner/moves

@@ -1 +1 @@
-��movesٴ{"0":{},"1":{"ORG":5064,"LOC":4932,"PER":4078},"2":{"ORG":5064,"LOC":4932,"PER":4078},"3":{"ORG":5064,"LOC":4932,"PER":4078},"4":{"ORG":5064,"LOC":4932,"PER":4078,"":1},"5":{"":1}}�cfg��neg_key�
+��movesٴ{"0":{},"1":{"LOC":5568,"ORG":5550,"PER":4662},"2":{"LOC":5568,"ORG":5550,"PER":4662},"3":{"LOC":5568,"ORG":5550,"PER":4662},"4":{"LOC":5568,"ORG":5550,"PER":4662,"":1},"5":{"":1}}�cfg��neg_key�

ner_transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:022d8220abeb5033522347c50e4b27ed090f01329dbec63b0a656bb81e63df1f
+oid sha256:8498589f7afcfecda66a6eb6c61bf466a18bf087d3ef8208507a60a2ecbec3e1
 size 440761243

use_custom_tokenizer.py

@@ -24,17 +24,26 @@ EXTENDED_LETTER_RANGE = "A-Za-zäöüÄÖÜàòèéìù"
 DATE = r"[0-3][1-9]\.[0-1][1-9]\.[1-2][0-9]{3}"
 TOP_LEVEL_DOMAINS = "ch|at|de|com|edu|org|gov|net|fr|uk|be|es|pl|it|eu|nl|ba|cz|dk|al|ad|bg|by|fi|gr|ie|li|lu|no|pt|ro|rs|ru|se|si|sk"
 
+DOT_BEFORE_WORD = [
+    rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{{i}}})(?!(({TOP_LEVEL_DOMAINS})\b))"
+    for i in range(3, 30)
+]
+
+
 DOT_AFTER_WORD = [
-    rf"(?<!www\.)(?<=([a-zA-ZäöüÄÖÜ]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
+    rf"(?<!www\.)(?<=([{EXTENDED_LETTER_RANGE}]){{{i}}})\.(?!({TOP_LEVEL_DOMAINS}))"
     for i in range(3, 30)
 ]
 
+
 DOT_AFTER_DATE = rf"(?<=({DATE}))\."
 
 infix_res = [
-    r"[\(\[\]\)]",
+    r"[\(\[\]\)\,\»\>\«\<]",  # brackets, commas and ' that are without space
     r"(?<=\.--)\.",  # DOT after .--
-    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    r"(?<=\,)\'",
+    *DOT_BEFORE_WORD,  # DOT before word
+    # rf"(?<!www)\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",
     r"'\.\.",  # e.g., 'Tscheicha'.. -> "Tscheicha" "'..", then split ".." as suffix
     *DOT_AFTER_WORD,  # when there is no space after the dot
     r"[A-Z](?=\. )",  # DOT after capital letter
@@ -45,6 +54,11 @@ LETTER_DOUBLE_ENDING_DOT_VAR_LENGTH = [  # DOT after letter, e.g., A.G., or u.s.
     rf"(?<=([{EXTENDED_LETTER_RANGE}]\.){{{i}}})\." for i in range(1, 30)
 ]
 
+prefix_res = [
+    rf"\.(?=[{EXTENDED_LETTER_RANGE}]{{3,20}})",  # DOT before word
+    *DOT_BEFORE_WORD,  # DOT before word
+]
+
 suffix_res = [
     r"(?<=\d)[\.]",  # DOT after number
     r"(?<=[\.])[\]\)]",  # Closing brackets with DOT before
@@ -63,10 +77,12 @@ NUMBER_SIGN_NUMBER_FIXED = r"(?<=[0-9])[+\*^](?=[0-9])"
 
 
 # Given a nlp object, return a custom tokenizer that splits on special cases and with unwanted tokenization removed
-def custom_tokenizer(nlp):
+def custom_tokenizer(nlp) -> Tokenizer:
     nlp.tokenizer = Tokenizer(nlp.vocab)
 
-    prefix_regex = compile_prefix_regex(nlp.Defaults.prefixes)
+    prefixes = nlp.Defaults.prefixes
+    prefixes += prefix_res
+    prefix_regex = compile_prefix_regex(prefixes)
     nlp.tokenizer.prefix_search = prefix_regex.search
 
     # We use the default infixes and remove some cases that lead to unwanted tokenization.