Update spaCy pipeline

README.md

@@ -35,15 +35,17 @@ model-index:
 
 | Type | Score |
 | --- | --- |
-| `REL_MICRO_P` | 54.47 |
-| `REL_MICRO_R` | 13.27 |
-| `REL_MICRO_F` | 21.35 |
-| `REL_MACRO_F` | 8.61 |
-| `REL_WEIGHTED_F` | 17.96 |
+| `REL_MICRO_P` | 46.91 |
+| `REL_MICRO_R` | 15.40 |
+| `REL_MICRO_F` | 23.18 |
+| `REL_MACRO_F` | 12.91 |
+| `REL_WEIGHTED_F` | 21.04 |
 | `F1_PART-OF` | 37.96 |
-| `F1_LOCATED-AT` | 0.00 |
-| `F1_CONNECTED-WITH` | 0.00 |
+| `F1_LOCATED-AT` | 12.87 |
+| `F1_CONNECTED-WITH` | 5.75 |
 | `F1_IN-MANNER-OF` | 0.00 |
-| `F1_ATTRIBUTE-FOR` | 5.09 |
-| `TRANSFORMER_LOSS` | 6.31 |
-| `RELATION_EXTRACTOR_LOSS` | 271.72 |
+| `F1_ATTRIBUTE-FOR` | 7.94 |
+| `F1_MACRO` | 0.00 |
+| `F1_WEIGHTED` | 0.00 |
+| `TRANSFORMER_LOSS` | 2.90 |
+| `RELATION_EXTRACTOR_LOSS` | 132.27 |

config.cfg

@@ -17,7 +17,7 @@ before_creation = null
 after_creation = null
 after_pipeline_creation = null
 tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
-batch_size = 300
+batch_size = 200
 vectors = {"@vectors":"spacy.Vectors.v1"}
 
 [components]
@@ -41,7 +41,7 @@ pooling = {"@layers":"reduce_mean.v1"}
 
 [components.relation_extractor.model.create_instance_tensor.get_instances]
 @misc = "rel_instance_generator.v1"
-max_length = 200
+max_length = 100
 
 [components.relation_extractor.model.create_instance_tensor.tok2vec]
 @architectures = "spacy-transformers.TransformerListener.v1"
@@ -51,7 +51,7 @@ upstream = "*"
 
 [components.transformer]
 factory = "transformer"
-max_batch_items = 4096
+max_batch_items = 2096
 set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
 
 [components.transformer.model]
@@ -88,8 +88,8 @@ dropout = 0.2
 accumulate_gradient = 1
 patience = 1600000
 max_epochs = 0
-max_steps = 20000
-eval_frequency = 100
+max_steps = 5000
+eval_frequency = 50
 frozen_components = []
 dev_corpus = "corpora.dev"
 train_corpus = "corpora.train"
@@ -127,14 +127,16 @@ initial_rate = 0.00005
 [training.score_weights]
 rel_micro_p = 0.0
 rel_micro_r = 0.0
-rel_micro_f = 0.12
-rel_macro_f = 0.12
-rel_weighted_f = 0.12
-f1_PART-OF = 0.12
-f1_LOCATED-AT = 0.12
-f1_CONNECTED-WITH = 0.12
-f1_IN-MANNER-OF = 0.12
-f1_ATTRIBUTE-FOR = 0.12
+rel_micro_f = 0.1
+rel_macro_f = 0.1
+rel_weighted_f = 0.1
+f1_PART-OF = 0.1
+f1_LOCATED-AT = 0.1
+f1_CONNECTED-WITH = 0.1
+f1_IN-MANNER-OF = 0.1
+f1_ATTRIBUTE-FOR = 0.1
+f1_macro = 0.1
+f1_weighted = 0.1
 
 [pretraining]
 

meta.json

@@ -39,18 +39,20 @@
 
   ],
   "performance":{
-    "rel_micro_p":0.5447470817,
-    "rel_micro_r":0.1327328751,
-    "rel_micro_f":0.2134553078,
-    "rel_macro_f":0.0861129219,
-    "rel_weighted_f":0.1795895562,
+    "rel_micro_p":0.4690909091,
+    "rel_micro_r":0.1539746956,
+    "rel_micro_f":0.2318475917,
+    "rel_macro_f":0.1290549706,
+    "rel_weighted_f":0.2104325211,
     "f1_PART-OF":0.3796196627,
-    "f1_LOCATED-AT":0.0,
-    "f1_CONNECTED-WITH":0.0,
+    "f1_LOCATED-AT":0.1286863271,
+    "f1_CONNECTED-WITH":0.0575296108,
     "f1_IN-MANNER-OF":0.0,
-    "f1_ATTRIBUTE-FOR":0.0509449466,
-    "transformer_loss":0.0630906408,
-    "relation_extractor_loss":2.7171609234
+    "f1_ATTRIBUTE-FOR":0.0794392523,
+    "f1_macro":0.0,
+    "f1_weighted":0.0,
+    "transformer_loss":0.0290014347,
+    "relation_extractor_loss":1.322729256
   },
   "requirements":[
     "spacy-transformers>=1.3.8,<1.4.0",

relationFactory.py

@@ -1,29 +1,34 @@
+from itertools import islice
 from typing import Tuple, List, Iterable, Optional, Dict, Callable, Any
 
+from spacy.scorer import PRFScore
+from thinc.types import Floats2d
+import numpy
+from spacy.training.example import Example
+from thinc.api import Model, Optimizer
+from spacy.tokens.doc import Doc
+from spacy.pipeline.trainable_pipe import TrainablePipe
+from spacy.vocab import Vocab
+from spacy import Language
+from thinc.model import set_dropout_rate
+from wasabi import Printer
+
+from typing import List, Tuple, Callable
+
 import spacy
 from spacy.tokens import Doc, Span
 from thinc.types import Floats2d, Ints1d, Ragged, cast
-from thinc.api import Model, Linear, chain, Logistic, Optimizer
+from thinc.api import Model, Linear, chain, Logistic
 
 import json
 import os
 import time
-from itertools import islice
 from pathlib import Path
 
 from sklearn.metrics import precision_recall_fscore_support, f1_score
-import numpy
-from spacy.training.example import Example
-from spacy.tokens.doc import Doc
-from spacy.pipeline.trainable_pipe import TrainablePipe
-from spacy.vocab import Vocab
-from spacy import Language
-from thinc.model import set_dropout_rate
-from wasabi import Printer
 import plotly.express as px
 import plotly.graph_objects as go
 
-
 @spacy.registry.architectures("rel_model.v1")
 def create_relation_model(
     create_instance_tensor: Model[List[Doc], Floats2d],
@@ -265,6 +270,17 @@ class RelationExtractor(TrainablePipe):
             self.set_annotations(docs, predictions)
         return losses
 
+    def get_focal_loss(self, examples: Iterable[Example], scores, gamma=3.0, alpha=0.25, eps=1e-8) -> Tuple[float, float]:
+        truths = self._examples_to_truth(examples)
+        scores_2 = numpy.clip(scores, eps, 1. - eps)
+        p_t = numpy.clip(scores_2 * truths + (1 - scores_2) * (1 - truths), eps, 1. - eps)
+
+        focal_loss = -(1 - p_t) ** gamma * numpy.log(p_t)
+        loss = numpy.mean(numpy.sum(focal_loss, axis=1))
+        gradient = focal_loss * (1 - 2 * truths)
+        return float(loss), gradient
+    
+
     def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]:
         """Find the loss and gradient of loss for the batch of documents and
         their predicted scores."""
@@ -452,4 +468,3 @@ def score_relations(examples: Iterable[Example], threshold: float) -> Dict[str,
     result["rel_weighted_f"] = f1_score(y_true, y_pred, average="weighted", labels=labels, zero_division=0)
 
     return result
-

ru_patents_rel_tiny-any-py3-none-any.whl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9a8bc5b8b749e4abfea1d8f93625b03fa857b049a9c1116f6aa0577ea141b82
-size 108767847
+oid sha256:4054040e76b605f22e2513c94ab0a96dc601bdd49d0defdd85a3ace67f830aea
+size 108770148

transformer/cfg

@@ -1,3 +1,3 @@
 {
-  "max_batch_items":4096
+  "max_batch_items":2096
 }

transformer/model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1281b5d341bd033984cf7f1fc5841b036641ae35a698beebbd2bc15bfca0f29d
-size 120293881
+oid sha256:35244535987be2a96005cdc63d93ae46ff5c0ab749f47435c212c65b94d176a2
+size 120294214