Upload 27 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+spacy-skmodel/sources/skner/wikiann-sk.bio filter=lfs diff=lfs merge=lfs -text

spacy-skmodel/.gitignore

@@ -0,0 +1,6 @@
+build
+venv
+dist
+input
+posparser
+nerposparser

spacy-skmodel/Makefile

@@ -0,0 +1,30 @@
+all: input/sk_snk-ud-test.spacy input/sk_snk-ud-train.spacy input/train-ner.spacy input/vectors/config.cfg
+
+sources/slovak-treebank/sk_snk-ud-test.conllu:
+	mkdir -p sources/slovak-treebank
+	cd sources && wget -P slovak-treebank https://raw.githubusercontent.com/UniversalDependencies/UD_Slovak-SNK/master/sk_snk-ud-test.conllu
+
+sources/slovak-treebank/sk_snk-ud-train.conllu:
+	mkdir -p sources/slovak-treebank
+	cd sources && wget -P slovak-treebank https://raw.githubusercontent.com/UniversalDependencies/UD_Slovak-SNK/master/sk_snk-ud-train.conllu
+
+sources/floret/vectors.floret.gz:
+	mkdir -p sources/floret
+	cd sources && wget -P floret https://files.kemt.fei.tuke.sk/models/fasttext/sk-fastext-floretvec-skweb2021/vectors.floret.gz  --no-check-certificate
+
+input/sk_snk-ud-test.spacy: sources/slovak-treebank/sk_snk-ud-test.conllu
+	mkdir -p input
+	spacy convert -n 10 sources/slovak-treebank/sk_snk-ud-test.conllu input
+
+input/sk_snk-ud-train.spacy: sources/slovak-treebank/sk_snk-ud-train.conllu
+	mkdir -p input
+	spacy convert -n 10 sources/slovak-treebank/sk_snk-ud-train.conllu input
+
+input/train-ner.spacy: sources/skner/wikiann-sk.bio
+	python skner2json.py ./sources/skner/wikiann-sk.bio input/train-ner.json input/test-ner.json
+	spacy convert input/train-ner.json input
+	spacy convert input/test-ner.json input
+
+input/vectors/config.cfg: sources/floret/vectors.floret.gz
+	mkdir -p input/vectors
+	spacy init vectors sk sources/floret/vectors.floret.gz input/vectors -V -m floret

spacy-skmodel/README.md

@@ -0,0 +1,70 @@
+# Slovak Spacy Model
+
+This is Slovak Spacy model.
+
+## Features
+
+- Requires Spacy 3.x.
+- Contains Floret Word Vectors.
+- Tagger module uses Slovak National Corpus Tagset.
+- Morphological analyzer uses Universal dependencies tagset and is trained on Slovak dependency treebank.
+- Lemmatizer is trained on Slovak dependency treebank.
+- Named entity recognizer is trained separately on WikiAnn database.
+
+
+## Downloads 
+
+# Version 3.4
+
+- [Spacy 3.4, Dependencies](https://files.kemt.fei.tuke.sk/models/spacy/sk_dep_web_md-3.4.1.tar.gz). 
+    - Model for trained lemmatization, POS tagging and dependency relations. 
+    - Contains Floret Word Vectors, trained on our web corpus.
+    - Should be without license issues.
+
+- [Spacy 3.4, NER + Dependencies](https://files.kemt.fei.tuke.sk/models/spacy/sk_core_web_md-3.4.1.tar.gz). 
+    - Includes the dependencies model.
+    - This model uses separate fine-tuned model for NER recognition. 
+
+# Version 3.3
+
+- [Spacy 3.3, Dependencies](https://files.kemt.fei.tuke.sk/models/spacy/sk_dep_web_md-3.3.0.tar.gz). Model for trained lemmatization, POS tagging and dependency relations. 
+- [Spacy 3.3, NER + Dependencies](https://files.kemt.fei.tuke.sk/models/spacy/sk_core_web_md-3.3.0.tar.gz). This model uses separate fine-tuned model for NER recognition. 
+
+These models do not have word vectors. 
+
+## Training
+
+Requirements for training:
+
+- Anaconda virtual environment
+- Spacy 3
+- make
+- bash
+
+Usage:
+
+1. Install dependencies in the Conda
+
+    ./prepare-env.sh
+
+2. Download and prepare data:
+
+    make
+
+3. Train models
+
+    ./train.sh
+
+## Credits 
+
+Author:
+
+Daniel Hládek daniel.hladek@tuke.sk and Technical University of Košice
+
+Sources:
+
+- The model uses spacy-transformers and [SlovakBERT](https://huggingface.co/gerulata/slovakbert).
+- [Part of Speech and Dependency relations](https://github.com/UniversalDependencies/UD_Slovak-SNK)
+The Slovak UD treebank with  Creative Commons - Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)
+- [Semi-automatic named entities](https://huggingface.co/datasets/wikiann) -  Unspecified License
+

spacy-skmodel/changemeta.py

@@ -0,0 +1,35 @@
+import json
+import sys
+
+pos_dname = sys.argv[1]
+with open(pos_dname + "/meta.json") as f:
+    pos_meta = json.load(f)
+    pos_performance = pos_meta["performance"]
+
+
+dname = sys.argv[2]
+meta_name = dname + "/meta.json"
+with open(meta_name) as f:
+    doc = json.load(f)
+    doc["name"] = "core_web_md"
+    if "disabled" in doc:
+        del doc["disabled"]
+    doc["pipeline"] = ["transformer","tagger","morphologizer","trainable_lemmatizer","parser","ner"]
+    for k,v in pos_performance.items():
+        doc["performance"][k] = v
+
+with open(meta_name,"w") as f:
+    json.dump(doc,f,indent=4)
+
+clines = []
+config_name = dname + "/config.cfg"
+with open(config_name) as f:
+    for l in f:
+        line = l.rstrip()
+        if "disabled" in line:
+            line = "disabled: []"
+        clines.append(line)
+
+
+with open(config_name,"w") as f:
+    print("\n".join(clines),file=f)

spacy-skmodel/clean.sh

@@ -0,0 +1,4 @@
+rm -rf traindir
+rm -rf posparser
+rm -rf nerposparser
+rm -rf dist

spacy-skmodel/config-ner.cfg

@@ -0,0 +1,160 @@
+[paths]
+train = "input/train-ner.spacy"
+dev = "input/test-ner.spacy"
+vectors = "input/vectors"
+init_tok2vec = null
+
+[system]
+gpu_allocator = null
+seed = 0
+
+[nlp]
+lang = "sk"
+pipeline = ["tok2vec","parser","tagger","ner"]
+batch_size = 1000
+#disabled = ["parser","tagger"]
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+[components]
+
+[components.ner]
+factory = "ner"
+moves = null
+update_with_oracle_cut_size = 100
+
+[components.ner.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+nO = null
+
+[components.ner.model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+
+[components.parser]
+source = "sk_pipeline"
+replace_listeners = ["model.tok2vec"]
+
+[components.tagger]
+source = "sk_pipeline"
+replace_listeners = ["model.tok2vec"]
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
+rows = [5000,2500,2500,2500]
+include_static_vectors = true
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+accumulate_gradient = 1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = ["tagger","parser"]
+before_to_disk = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+get_length = null
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+t = 0.0
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+learn_rate = 0.001
+
+[training.score_weights]
+dep_las_per_type = null
+sents_p = null
+sents_r = null
+ents_per_type = null
+dep_uas = 0.17
+dep_las = 0.17
+sents_f = 0.0
+tag_acc = 0.33
+ents_f = 0.33
+ents_p = 0.0
+ents_r = 0.0
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

spacy-skmodel/config-transformer-ner.cfg

@@ -0,0 +1,165 @@
+[paths]
+train = "input/train-ner.spacy"
+dev = "input/test-ner.spacy"
+vectors = "input/vectors"
+init_tok2vec = null
+
+[system]
+gpu_allocator = "pytorch"
+seed = 0
+
+[nlp]
+lang = "sk"
+pipeline = ["transformer","tagger","morphologizer","trainable_lemmatizer","parser","ner"]
+batch_size = 128
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+[components]
+
+[components.ner]
+factory = "ner"
+
+[components.ner.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = false
+nO = null
+
+[components.ner.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+
+[components.ner.model.tok2vec.pooling]
+@layers = "reduce_mean.v1"
+
+[components.morphologizer]
+source = "sk_dep_web_md"
+replace_listeners = ["model.tok2vec"]
+
+[components.parser]
+source = "sk_dep_web_md"
+replace_listeners = ["model.tok2vec"]
+
+[components.tagger]
+source = "sk_dep_web_md"
+replace_listeners = ["model.tok2vec"]
+
+[components.trainable_lemmatizer]
+source = "sk_dep_web_md"
+replace_listeners = ["model.tok2vec"]
+
+[components.transformer]
+factory = "transformer"
+max_batch_items = 4096
+set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v3"
+name = "gerulata/slovakbert"
+mixed_precision = false
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.transformer.model.grad_scaler_config]
+
+[components.transformer.model.tokenizer_config]
+use_fast = true
+
+[components.transformer.model.transformer_config]
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = ["tagger","morphologizer","trainable_lemmatizer","parser"]
+annotating_components = []
+before_to_disk = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+get_length = null
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 0.00005
+
+[training.score_weights]
+tag_acc = 0.26
+pos_acc = 0.12
+morph_acc = 0.12
+morph_per_feat = null
+lemma_acc = 0.26
+dep_uas = 0.12
+dep_las = 0.12
+dep_las_per_type = null
+sents_p = null
+sents_r = null
+sents_f = 0.0
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.tokenizer]

spacy-skmodel/config-transformer.cfg

@@ -0,0 +1,200 @@
+[paths]
+train = "input/sk_snk-ud-train.spacy"
+dev = "input/sk_snk-ud-test.spacy"
+vectors = "input/vectors"
+init_tok2vec = null
+
+[system]
+gpu_allocator = "pytorch"
+seed = 0
+
+[nlp]
+lang = "sk"
+pipeline = ["transformer","tagger","morphologizer","trainable_lemmatizer","parser"]
+batch_size = 128
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+[components]
+
+[components.morphologizer]
+factory = "morphologizer"
+extend = false
+overwrite = true
+scorer = {"@scorers":"spacy.morphologizer_scorer.v1"}
+
+[components.morphologizer.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+normalize = false
+
+[components.morphologizer.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+pooling = {"@layers":"reduce_mean.v1"}
+upstream = "*"
+
+[components.parser]
+factory = "parser"
+learn_tokens = false
+min_action_freq = 30
+moves = null
+scorer = {"@scorers":"spacy.parser_scorer.v1"}
+update_with_oracle_cut_size = 100
+
+[components.parser.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "parser"
+extra_state_tokens = false
+hidden_width = 128
+maxout_pieces = 3
+use_upper = false
+nO = null
+
+[components.parser.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+pooling = {"@layers":"reduce_mean.v1"}
+upstream = "*"
+
+[components.tagger]
+factory = "tagger"
+neg_prefix = "!"
+overwrite = false
+scorer = {"@scorers":"spacy.tagger_scorer.v1"}
+
+[components.tagger.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+normalize = false
+
+[components.tagger.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+pooling = {"@layers":"reduce_mean.v1"}
+upstream = "*"
+
+[components.trainable_lemmatizer]
+factory = "trainable_lemmatizer"
+backoff = "orth"
+min_tree_freq = 3
+overwrite = false
+scorer = {"@scorers":"spacy.lemmatizer_scorer.v1"}
+top_k = 1
+
+[components.trainable_lemmatizer.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+normalize = false
+
+[components.trainable_lemmatizer.model.tok2vec]
+@architectures = "spacy-transformers.TransformerListener.v1"
+grad_factor = 1.0
+pooling = {"@layers":"reduce_mean.v1"}
+upstream = "*"
+
+[components.transformer]
+factory = "transformer"
+max_batch_items = 4096
+set_extra_annotations = {"@annotation_setters":"spacy-transformers.null_annotation_setter.v1"}
+
+[components.transformer.model]
+@architectures = "spacy-transformers.TransformerModel.v3"
+name = "gerulata/slovakbert"
+mixed_precision = false
+
+[components.transformer.model.get_spans]
+@span_getters = "spacy-transformers.strided_spans.v1"
+window = 128
+stride = 96
+
+[components.transformer.model.grad_scaler_config]
+
+[components.transformer.model.tokenizer_config]
+use_fast = true
+
+[components.transformer.model.transformer_config]
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+accumulate_gradient = 3
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.1
+patience = 1600
+max_epochs = 0
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+annotating_components = []
+before_to_disk = null
+
+[training.batcher]
+@batchers = "spacy.batch_by_padded.v1"
+discard_oversize = true
+size = 2000
+buffer = 256
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+
+[training.optimizer.learn_rate]
+@schedules = "warmup_linear.v1"
+warmup_steps = 250
+total_steps = 20000
+initial_rate = 0.00005
+
+[training.score_weights]
+tag_acc = 0.26
+pos_acc = 0.12
+morph_acc = 0.12
+morph_per_feat = null
+lemma_acc = 0.26
+dep_uas = 0.12
+dep_las = 0.12
+dep_las_per_type = null
+sents_p = null
+sents_r = null
+sents_f = 0.0
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+
+[initialize.components]
+
+[initialize.tokenizer]

spacy-skmodel/meta.json

@@ -0,0 +1,10 @@
+{
+  "lang":"sk",
+  "name":"dep_web_md",
+  "version":"3.4.1",
+  "description":"Slovak model with part-of-speech and parsing",
+  "author":"Daniel Hládek",
+  "email":"daniel.hladek@tuke.sk",
+  "url":"https://nlp.kemt.fei.tuke.sk",
+  "license":"BSD"
+}

spacy-skmodel/prepare-env.sh

@@ -0,0 +1,3 @@
+conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
+pip install -U spacy[cuda113,transformers,lookups]==3.4
+rm -r ./input/*

spacy-skmodel/skner2json.py

@@ -0,0 +1,103 @@
+import sys
+import json
+# https://spacy.io/api/data-formats#training
+#from spacy.gold import offsets_from_biluo_tags
+#from spacy.gold import iob_to_biluo
+
+def bio2bliou(ners):
+    state = 0
+    ners1 = []
+    # add U
+    for i,ner in enumerate(ners):
+        ners1.append(list(ner))
+        if i > 0 and ners[i-1][0] != "B" and ners[i-1][0]!= "I" and ner[0] == "I":
+            ners1[i][0] = "B"
+            print("fixed")
+    ners = ners1
+    ners1 = []
+    for i,ner in enumerate(ners):
+        ners1.append(ner)
+        if i > 0 and ners[i-1][0] == "B" and ner[0] != "I" and ner !="O":
+            ners1[i-1][0] = "U"
+        if i > 1 and (ners[i-2][0] == "I" or ners[i-2][0] == "B") and ners[i-1][0] == "I" and ners[i][0] != "I":
+            ners1[i-1][0] = "L"
+    if len(ners) == 1 and ners[0][0] == "I":
+        ners1[0][0] = "U"
+    if len(ners) > 1 and ners[-1][0] == "B":
+        ners1[-1][0] = "U"
+    if len(ners) > 0 and ners[-1][0] == "I":
+        ners1[-1][0] = "L"
+    ners2 = []
+    for nerlist in ners1:
+        ners2.append("".join(nerlist))
+    #if len(ners2) == 2:
+    return ners2
+
+def save_sentences(sentences,filename):
+    paragraphs = []
+    for id,sentence in enumerate(sentences):
+        tokens = []
+        words = []
+        for word,tag in sentence:
+            words.append(word)
+            tokens.append({"orth":word,"ner":tag})
+        paragraphs.append({"id":id,"paragraphs":[{"raw":" ".join(words),"sentences":[{"tokens":tokens}]}]})
+    with open(filename,"w") as f:
+        json.dump(paragraphs,f)
+
+
+def strippunct(word):
+    chars = list(word)
+    repl = "\"' ,.()"
+    if not word[0].isalpha():
+        chars[0] = "x"
+    if not word[-1].isalpha():
+        chars[-1] = "x"
+    #if not word.isalpha():
+    #    print(word)
+    #for c in word:
+    #    if c in repl:
+    #        c="x"
+    #    chars.append(c)
+    return "".join(chars)
+
+def process_data(filename):
+    with open(filename) as f:
+        sentences = []
+        words = []
+        ners = []
+        for l in f:
+            line = l.strip()
+            if len(line) > 0:
+                tokens = l.split()
+                word = tokens[0].strip()
+                ner = tokens[-1].strip()
+                #word = strippunct(word)
+                if len(ner) > 1 and ner[1] == "-":
+                    word = strippunct(word)
+                    if len(word) == 0:
+                        continue
+                words.append(word)
+                ners.append(ner)
+            else:
+                #print(ners)
+                ners = bio2bliou(ners)
+                sentence = []
+                for word,tag in zip(words,ners):
+                    sentence.append((word,tag))
+                #print(sentence)
+                sentences.append(sentence)
+                del ners[:]
+                del words[:]
+    testset = []
+    trainset = []
+    for i,sentence in enumerate(sentences):
+        if i % 10 == 0:
+            testset.append(sentence)
+        else:
+            trainset.append(sentence)
+
+    save_sentences(trainset,sys.argv[2])
+    save_sentences(testset,sys.argv[3])
+
+process_data(sys.argv[1])

spacy-skmodel/small-config.cfg

@@ -0,0 +1,189 @@
+[paths]
+train = "input/sk_snk-ud-train.spacy"
+dev = "input/sk_snk-ud-test.spacy"
+vectors = null
+init_tok2vec = null
+
+[system]
+gpu_allocator = "pytorch"
+seed = 0
+
+[nlp]
+lang = "sk"
+pipeline = ["tok2vec","tagger","morphologizer","trainable_lemmatizer","parser"]
+batch_size = 1000
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+[components]
+
+[components.morphologizer]
+factory = "morphologizer"
+
+[components.morphologizer.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+
+[components.morphologizer.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+[components.parser]
+factory = "parser"
+
+[components.parser.model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "parser"
+extra_state_tokens = false
+hidden_width = 128
+maxout_pieces = 3
+use_upper = true
+nO = null
+
+[components.parser.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+
+[components.tagger]
+factory = "tagger"
+
+[components.tagger.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+
+[components.tagger.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+
+
+[components.tok2vec]
+factory = "tok2vec"
+
+[components.tok2vec.model]
+@architectures = "spacy.Tok2Vec.v2"
+
+[components.tok2vec.model.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = ${components.tok2vec.model.encode.width}
+attrs = ["NORM","PREFIX","SUFFIX","SHAPE"]
+rows = [5000,2500,2500,2500]
+include_static_vectors = false
+
+[components.tok2vec.model.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 96
+depth = 4
+window_size = 1
+maxout_pieces = 3
+
+[components.trainable_lemmatizer]
+factory = "trainable_lemmatizer"
+backoff = "orth"
+min_tree_freq = 3
+overwrite = false
+scorer = {"@scorers":"spacy.lemmatizer_scorer.v1"}
+top_k = 1
+
+[components.trainable_lemmatizer.model]
+@architectures = "spacy.Tagger.v2"
+nO = null
+normalize = false
+
+[components.trainable_lemmatizer.model.tok2vec]
+@architectures = "spacy.Tok2VecListener.v1"
+width = ${components.tok2vec.model.encode.width}
+
+[corpora]
+
+[corpora.dev]
+@readers = "spacy.Corpus.v1"
+path = ${paths.dev}
+max_length = 0
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[corpora.train]
+@readers = "spacy.Corpus.v1"
+path = ${paths.train}
+max_length = 2000
+gold_preproc = false
+limit = 0
+augmenter = null
+
+[training]
+dev_corpus = "corpora.dev"
+train_corpus = "corpora.train"
+seed = ${system.seed}
+gpu_allocator = ${system.gpu_allocator}
+dropout = 0.25
+accumulate_gradient = 1
+patience = 1600
+max_epochs = 25
+max_steps = 20000
+eval_frequency = 200
+frozen_components = []
+before_to_disk = null
+annotating_components = []
+
+[training.batcher]
+@batchers = "spacy.batch_by_words.v1"
+discard_oversize = false
+tolerance = 0.2
+get_length = null
+
+[training.batcher.size]
+@schedules = "compounding.v1"
+start = 100
+stop = 1000
+compound = 1.001
+t = 0.0
+
+[training.logger]
+@loggers = "spacy.ConsoleLogger.v1"
+progress_bar = false
+
+[training.optimizer]
+@optimizers = "Adam.v1"
+beta1 = 0.9
+beta2 = 0.999
+L2_is_weight_decay = true
+L2 = 0.01
+grad_clip = 1.0
+use_averages = false
+eps = 0.00000001
+learn_rate = 0.001
+
+[training.score_weights]
+tag_acc = 0.17
+pos_acc = 0.17
+morph_acc = 0.17
+morph_per_feat = null
+lemma_acc = 0.33
+dep_uas = 0.08
+dep_las = 0.08
+dep_las_per_type = null
+sents_p = null
+sents_r = null
+sents_f = 0.0
+
+[pretraining]
+
+[initialize]
+vectors = ${paths.vectors}
+init_tok2vec = ${paths.init_tok2vec}
+vocab_data = null
+lookups = null
+before_init = null
+after_init = null
+
+[initialize.components]
+
+[initialize.components.tagger]
+
+[initialize.tokenizer]

spacy-skmodel/sources/skner/README.txt

@@ -0,0 +1,12 @@
+Silver-standard Name Annotations From Wikipedia Markups
+Xiaoman Pan
+panx2@rpi.edu
+
+FORMAT:
+[TOKEN] [ADDITIONAL INFORMATION] [TAG]
+
+ADDITIONAL INFORMATION FORMAT:
+[Wikipedia title] [name mention] [entity type] [entity type confidence] [English Wikipedia title]
+
+If you would like to cite this work, please cite the following publication:
+Cross-lingual Name Tagging and Linking for 282 Languages

spacy-skmodel/sources/skner/wikiann-sk.bio

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c3ff1eb8ea5bf2de7a19f44d36df45e61a3f52d3e415b81dfeadeffc61ee4e
+size 13898246

spacy-skmodel/sources/ud-artificial-gapping/README.txt

@@ -0,0 +1,29 @@
+Artificial dependency trees in the Universal Dependencies v2 style, focused
+on gapping (the 'orphan' relation in UD). For motivation and description of
+the data, see the paper cited below. Please cite the paper if you use the data
+in your academic work.
+
+@inproceedings{droganova2018,
+  title     = {Parse Me if You Can: Artificial Treebanks for Parsing Experiments on Elliptical Constructions},
+  author    = {Kira Droganova and Daniel Zeman and Jenna Kanerva and Filip Ginter},
+  year      = {2018},
+  booktitle = {Proceedings of the 11th International Conference on Language Resources and Evaluation ({LREC} 2018)},
+  publisher = {European Language Resources Association},
+  organization = {European Language Resource Association},
+  address   = {Paris, France},
+  location  = {Miyazaki, Japan},
+  venue     = {Phoenix Seagaia Conference Center}
+}
+
+Permanent URI of the dataset:
+http://hdl.handle.net/11234/1-2616
+
+*-crawled-* data are crawled from the web, parsed by two parsers, filtered so
+    that only those trees survive where the two parsers agree, then proceesed
+    to create artificial gapping
+*-{train,dev,test}-* data are based on Universal Dependency treebanks release
+    2.1 (November 2017)
+English and Finnish data were manually checked and modified after gapping
+    structures had been automatically drafted.
+Czech, Slovak and Russian data were processed only automatically.
+

spacy-skmodel/testmodel.py

@@ -0,0 +1,13 @@
+import spacy
+import sys
+
+nlp = spacy.load(sys.argv[1])
+nlp.enable_pipe("tagger")
+nlp.enable_pipe("parser")
+nlp.enable_pipe("ner")
+lines = []
+for line in sys.stdin:
+    lines.append(line.rstrip())
+doc = nlp("\n".join(lines))
+for token in doc:
+    print(token.text, token.lemma_, token.pos_, token.tag_, token.dep_,token.shape_, token.is_alpha, token.is_stop,token.ner_)

spacy-skmodel/train-small.sh

@@ -0,0 +1,35 @@
+set -e # fail on error
+
+#make # prepare data
+
+export CUDA_VISIBLE_DEVICES=0
+# cleanup old results
+#rm -rf dist
+mkdir -p dist
+mkdir -p train
+TRAINDIR=train/smposparser
+NERTRAINDIR=train/smnerposparser
+VER=3.3.0
+MODELDIR=dist/sk_dep_web_sm-$VER
+NERMODELDIR=dist/sk_core_web_sm-$VER
+mkdir -p $TRAINDIR 
+# Train POS and dependencies
+spacy train small-config.cfg -o $TRAINDIR -g 0 > $TRAINDIR/train.log 2> $TRAINDIR/train.err.log
+# Package POS
+spacy package -m small-meta.json -F $TRAINDIR/model-best dist
+cd $MODELDIR
+python ./setup.py sdist
+# install to include pos and dependencies in new model
+# name must be the same as in meta.json
+pip install $MODELDIR.tar.gz
+cd ../../
+mkdir -p $NERTRAINDIR
+# Train NER, copy POS and dep from old model
+spacy train small-ner.cfg -o $NERTRAINDIR -g 0 > $NERTRAINDIR/train.log 2> $NERTRAINDIR/train.err.log
+# Correct meta
+cp $NERTRAINDIR/model-best/meta.json $NERTRAINDIR/model-best/meta-ner.json
+python changemeta.py $TRAINDIR/model-best $NERTRAINDIR/model-best
+# Package result
+spacy package --version $VER $NERTRAINDIR/model-best dist
+cd $NERMODELDIR
+python ./setup.py sdist

spacy-skmodel/train.sh

@@ -0,0 +1,31 @@
+set -e # fail on error
+
+make # prepare data
+
+export CUDA_VISIBLE_DEVICES=0
+VERSION=3.4.1
+# cleanup old results
+#rm -rf dist
+mkdir -p dist
+mkdir -p train
+mkdir -p train/sposparser
+# Train POS and dependencies
+spacy train config-transformer.cfg -o ./train/sposparser -g 0 > ./train/sposparser/train.log 2> ./train/sposparser/train.err.log
+# Package POS
+spacy package -m meta.json -F train/sposparser/model-best dist
+cd dist/sk_dep_web_md-$VERSION
+python ./setup.py sdist
+# install to include pos and dependencies in new model
+# name must be the same as in meta.json
+#pip install dist/sk_dep_web_md-$VERSION.tar.gz
+#cd ../../
+#mkdir -p train/snerposparser
+# Train NER, copy POS and dep from old model
+#spacy train config-transformer-ner.cfg -o ./train/snerposparser -g 0 > ./train/snerposparser/train.log 2> ./train/snerposparser/train.err.log
+# Correct meta
+#cp ./train/snerposparser/model-best/meta.json ./train/snerposparser/model-best/meta-ner.json
+#python changemeta.py ./train/sposparser/model-best ./train/snerposparser/model-best
+# Package result
+#spacy package --version $VERSION train/snerposparser/model-best dist
+#cd dist/sk_core_web_md-$VERSION 
+#python ./setup.py sdist

spacy-skmodel/treebank2json.py

@@ -0,0 +1,111 @@
+import sys
+import json
+# https://spacy.io/api/data-formats#training
+#from spacy.gold import offsets_from_biluo_tags
+#from spacy.gold import iob_to_biluo
+
+depmap = {
+"case":"AuxP",
+"root" : "Pred", # / Pred_M
+"punct" : "AuxK",
+"nsubj" : "Sb",
+"obj" : "Obj",
+"conj" : "Sb",
+"cc" : "Coord",
+"orphan" : "Obj",
+"advmod" : "Adv",
+"amod" : "Atr",
+"nmod" : "Atr",
+"mark" : "AuxC",
+"aux" : "AuxV",
+"det" : "Atr",
+"obl" : "Atr",
+"expl:pv" : "AuxT",
+"advmod" : "Adv",
+}
+
+def save_data(filename,dataset):
+    sentences = []
+    words = []
+    docs = []
+    for i,item in enumerate(dataset):
+        bad = False
+        for token in item:
+            words.append(token["orth"])
+            h = token["head"] + token["id"]
+            #print(h,len(item))
+            if h < 0 or  h >= len(item):
+                print(item)
+                bad = True
+                break
+        if bad:
+            continue
+        sentences.append({"tokens":item})
+        if len(sentences) > 4:
+            doc = {
+                "id": i,
+                "paragraphs":[{
+                    "raw": " ".join(words),
+                    "sentences": list(sentences)
+                }]
+            }
+            docs.append(doc)
+            del words[:]
+            del sentences[:]
+
+    if len(docs)> 0 and len(sentences)>0: 
+        doc = {
+            "id": docs[-1]["id"] + 1,
+            "paragraphs":[{
+                "raw": " ".join(words),
+                "sentences": list(sentences)
+            }]
+        }
+        docs.append(doc)
+    with open(filename,"w") as f:
+        json.dump(docs,f)
+
+
+def process_data(trainname,testname):
+    dataset = []
+    sentence = []
+    for l in sys.stdin:
+        if l[0] == "#":
+            continue
+        tokens = l.split()
+        #print(tokens)
+        if len(tokens)  < 2:
+            if len(sentence) > 0:
+                dataset.append(list(sentence))
+                del sentence[:]
+            continue
+        head = int(tokens[6])
+        id = int(tokens[0]) -1
+        print(head,id)
+        h = 0
+        if head != 0:
+            h = head - id -1
+        dep = tokens[7]
+        if dep in depmap:
+            dep = depmap[dep]
+        #print(h)
+        token = {
+            "id": id,
+            "orth": tokens[1],
+            "tag": tokens[4],
+            # "ner":
+            "head": h,
+            "dep": dep,
+        }
+        sentence.append(token)
+    trainset = []
+    testset = []
+    for i, item in enumerate(dataset):
+        if i % 10 == 0:
+            testset.append(item)
+        else:
+            trainset.append(item)
+    save_data(trainname,trainset)
+    save_data(testname,testset)
+
+process_data(sys.argv[1],sys.argv[2])

spacy-skmodel/v2/01.prepare.sh

@@ -0,0 +1,15 @@
+# conda install spacy=2.3.5 cupy cudatoolkit=9.2
+mkdir -p input
+# Prepare Treebank
+mkdir -p input/slovak-treebank
+spacy convert ./sources/slovak-treebank/stb.conll ./input/slovak-treebank
+# UDAG used as evaluation
+mkdir -p input/ud-artificial-gapping
+spacy convert ./sources/ud-artificial-gapping/sk-ud-crawled-orphan.conllu ./d/input/ud-artificial-gapping
+# Prepare skner
+mkdir -p input/skner
+cd input/skner
+python ../../skner2json.py ../../sources/skner/wikiann-sk.bio
+
+wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.sk.300.vec.gz
+mv cc.sk.300.vec.gz ./input

spacy-skmodel/v2/assemble.py

@@ -0,0 +1,39 @@
+import sys
+import json
+
+base = sys.argv[1]
+ner = sys.argv[2]
+posparser = sys.argv[3]
+outmeta = sys.argv[4]
+
+meta = None
+with open(base,"rb") as f:
+    meta = json.load(f)
+    meta["labels"] = {}
+    meta["accuracy"] = {}
+
+ner_meta = None
+with open(ner,"rb") as f:
+    ner_meta = json.load(f)
+    meta["spacy_version"] = ner_meta["spacy_version"]
+    meta["labels"]["ner"] = ner_meta["labels"]["ner"]
+    meta["accuracy"]["ents_p"] = ner_meta["accuracy"]["ents_p"]
+    meta["accuracy"]["ents_r"] = ner_meta["accuracy"]["ents_r"]
+    meta["accuracy"]["ents_f"] = ner_meta["accuracy"]["ents_f"]
+    meta["accuracy"]["ents_per_type"] = ner_meta["accuracy"]["ents_per_type"]
+
+posparser_meta = None
+with open(posparser,"rb") as f:
+    posparser_meta = json.load(f)
+    meta["vectors"] = posparser_meta["vectors"]
+    meta["accuracy"]["tags_acc"] = posparser_meta["accuracy"]["tags_acc"]
+    meta["accuracy"]["uas"] = posparser_meta["accuracy"]["uas"]
+    meta["accuracy"]["las"] = posparser_meta["accuracy"]["las"]
+    meta["accuracy"]["las_per_type"] = posparser_meta["accuracy"]["las_per_type"]
+    meta["labels"]["tagger"] = posparser_meta["labels"]["tagger"]
+
+with open(outmeta,"w") as f:
+    json.dump(meta,f,indent=6)
+
+
+

spacy-skmodel/v2/meta-ccv2.json

@@ -0,0 +1,12 @@
+{
+  "lang":"sk",
+  "name": "sk_core_web_lg",
+  "version": "2.3.1",
+  "description": "Basic Slovak model with fastext word vectors trained on public data",
+  "author":"Daniel Hládek",
+  "email":"dhladek@gmail.com",
+  "url":"https://nlp.kemt.fei.tuke.sk",
+  "license":"CC BY-SA 3.0",
+  "pipeline": ["tagger","parser","ner"]
+}
+

spacy-skmodel/v2/meta-v2.json

@@ -0,0 +1,12 @@
+{
+  "lang":"sk",
+  "name": "sk_core_web_md",
+  "version": "2.3.1",
+  "description": "Basic Slovak model without word vectors trained on public data",
+  "author":"Daniel Hládek",
+  "email":"dhladek@gmail.com",
+  "url":"https://nlp.kemt.fei.tuke.sk",
+  "license":"CC BY-SA 3.0",
+  "pipeline": ["tagger","parser","ner"]
+}
+

spacy-skmodel/v2/train-v2.sh

@@ -0,0 +1,21 @@
+FLAGS="--n-iter 10"
+OUTDIR=outv2
+rm -r $OUTDIR
+mkdir -p $OUTDIR
+# Train dependency and POS
+spacy train sk $OUTDIR/posparser input/slovak-treebank input/ud-artificial-gapping  -p tagger,parser $FLAGS
+# Train NER
+spacy train sk $OUTDIR/ner input/skner/train.json input/skner/test.json -p ner  -R $FLAGS
+
+## Assemle model
+mkdir -p $OUTDIR/nerposparser
+cp -r $OUTDIR/posparser/model-final/* $OUTDIR/nerposparser
+cp -r $OUTDIR/ner/model-final/ner $OUTDIR/nerposparser
+python ./assemble.py v2/meta-v2.json $OUTDIR/ner/model-final/meta.json $OUTDIR/posparser/model-final/meta.json $OUTDIR/nerposparser/meta.json
+
+# Make python package
+mkdir -p $OUTDIR/dist
+spacy package $OUTDIR/nerposparser $OUTDIR/dist
+DNAME=`ls $OUTDIR/dist`
+cd $OUTDIR/dist/$DNAME
+python ./setup.py sdist --dist-dir ../

spacy-skmodel/v2/train-v2cc.sh

@@ -0,0 +1,23 @@
+FLAGS="-g 0 --n-iter 10"
+OUTDIR=outccv2
+rm -r $OUTDIR
+mkdir -p $OUTDIR
+spacy init-model sk $OUTDIR/basic -v ./input/cc.sk.300.vec.gz  -V 600000
+
+# Train dependency and POS
+spacy train sk $OUTDIR/posparser input/slovak-treebank input/ud-artificial-gapping  -p tagger,parser -b $OUTDIR/basic $FLAGS 
+
+ spacy train sk $OUTDIR/ner input/skner/train.json input/skner/test.json -p ner  -R -b $OUTDIR/basic $FLAGS
+
+## Assemle model
+mkdir -p $OUTDIR/nerposparser
+cp -r $OUTDIR/posparser/model-final/* $OUTDIR/nerposparser
+cp -r $OUTDIR/ner/model-final/ner $OUTDIR/nerposparser
+python ./assemble.py v2/meta-ccv2.json $OUTDIR/ner/model-final/meta.json $OUTDIR/posparser/model-final/meta.json $OUTDIR/nerposparser/meta.json
+
+# Make python package
+mkdir -p $OUTDIR/dist
+spacy package $OUTDIR/nerposparser $OUTDIR/dist
+DNAME=`ls $OUTDIR/dist`
+cd $OUTDIR/dist/$DNAME
+python ./setup.py sdist --dist-dir ../