Update spaCy pipeline

README.md

@@ -1,3 +1,37 @@
 ---
+tags:
+- spacy
+language:
+- fr
 license: cc-by-nc-2.0
+model-index:
+- name: fr_arches_sentencizer
+  results:
+  - task:
+      name: SENTS
+      type: token-classification
+    metrics:
+    - name: Sentences F-Score
+      type: f_score
+      value: 0.4066841152
 ---
+| Feature | Description |
+| --- | --- |
+| **Name** | `fr_arches_sentencizer` |
+| **Version** | `0.0.0` |
+| **spaCy** | `>=3.6.1,<3.7.0` |
+| **Default Pipeline** | `preprocess_text`, `senter` |
+| **Components** | `preprocess_text`, `senter` |
+| **Vectors** | 0 keys, 0 unique vectors (0 dimensions) |
+| **Sources** | n/a |
+| **License** | `cc-by-nc 2.0` |
+| **Author** | [n/a]() |
+
+### Accuracy
+
+| Type | Score |
+| --- | --- |
+| `SENTS_F` | 40.67 |
+| `SENTS_P` | 52.91 |
+| `SENTS_R` | 33.03 |
+| `SENTER_LOSS` | 162448.50 |

config.cfg

@@ -0,0 +1,124 @@
+[paths]
+train = "drive/MyDrive/ARCHES/TAL/donnees/pretraitement/modele_segmentation/extraction_wavestone/spacy/train.spacy"
+dev = "drive/MyDrive/ARCHES/TAL/donnees/pretraitement/modele_segmentation/extraction_wavestone/spacy/dev.spacy"
+vectors = null
+init_tok2vec = null
+
+[system]
+gpu_allocator = "pytorch"
+seed = 0
+
+[nlp]
+lang = "fr"
+pipeline = ["preprocess_text","senter"]
+batch_size = 128
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+[components]
+
+[components.preprocess_text]
+factory = "preprocess_text"
+
+[components.senter]
+factory = "senter"
+overwrite = false
+scorer = {"@scorers":"spacy.senter_scorer.v1"}
+
+[components.senter.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+normalize = false
+
+[components.senter.model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 12
+depth = 1
+embed_size = 2000
+window_size = 1
+maxout_pieces = 2
+subword_features = true
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+annotating_components = []
+before_to_disk = null
+before_update = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+get_length = null
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 0.00005
+
+[training.score_weights]
+sents_f = 1.0
+sents_p = 0.0
+sents_r = 0.0
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

meta.json

@@ -0,0 +1,41 @@
+{
+  "lang":"fr",
+  "name":"arches_sentencizer",
+  "version":"0.0.0",
+  "description":"",
+  "author":"",
+  "email":"",
+  "url":"",
+  "license":"cc-by-nc 2.0",
+  "spacy_version":">=3.6.1,<3.7.0",
+  "spacy_git_version":"458bc5f45",
+  "vectors":{
+    "width":0,
+    "vectors":0,
+    "keys":0,
+    "name":null
+  },
+  "labels":{
+
+  },
+  "pipeline":[
+    "preprocess_text",
+    "senter"
+  ],
+  "components":[
+    "preprocess_text",
+    "senter"
+  ],
+  "disabled":[
+
+  ],
+  "performance":{
+    "sents_f":0.4066841152,
+    "sents_p":0.529072813,
+    "sents_r":0.3302812296,
+    "senter_loss":1624.4849906801
+  },
+  "requirements":[
+
+  ]
+}

preprocess_text.py

@@ -0,0 +1,41 @@
+import re
+from spacy.tokens import Doc
+from spacy.language import Language
+
+
+@Language.component("preprocess_text")
+
+def preprocess_text(doc):
+
+  text = doc.text
+
+  #On supprime les espaces présentes en fin de ligne
+  etape1_in = re.compile('\s$', re.MULTILINE)
+  etape1_out = ''
+
+  #On supprime les retours à la ligne au niveau des traits d'union au sein d'un mot
+  # "gallo-\romain" devient ainsi "gallo-romain"
+  etape2_in = re.compile('(([a-zà-ÿ]|[A-ZÀ-Ÿ])-)\n')
+  etape2_out = r'\1'
+
+  #On remplace les retours à la ligne restants par une espace
+  etape3_in = re.compile('\n')
+  etape3_out = ' '
+
+  #On remplace les espaces multiples par une espace unique
+  etape4_in = re.compile('\s{2,}')
+  etape4_out = ' '
+
+  #On applique les transformations
+  sortie1 = etape1_in.sub(etape1_out, text)
+  sortie2 = etape2_in.sub(etape2_out, sortie1)
+  sortie3 = etape3_in.sub(etape3_out, sortie2)
+  sortie4 = etape4_in.sub(etape4_out, sortie3)
+
+  # On crée un doc avec le texte modifié
+  modified_doc = Doc(doc.vocab, words=sortie4.split())
+
+  return modified_doc
+
+
+Language.component("preprocess_text", func=preprocess_text)

senter/cfg

@@ -0,0 +1,3 @@
+{
+  "overwrite":false
+}

vocab/key2row

@@ -0,0 +1 @@
+�

vocab/lookups.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76be8b528d0075f7aae98d6fa57a6d3c83ae480a8469e668d7b0af968995ac71
+size 1

vocab/vectors.cfg

@@ -0,0 +1,3 @@
+{
+  "mode":"default"
+}