Initial commit with adapted deliverables from Clarin: http://hdl.handle.net/20.500.12537/301

General:

- use submodules for electra model and Tokenizer package
- create missing directories
- add .python-version for 3.8 (same as for biaffine_parser)
- add .gitignore
- model file best.th in git-lfs

README.md:

- removed the Icelandic text because of redundancies



Signed-off-by: Daniel Schnell <dschnell@grammatek.com>

.gitignore

@@ -0,0 +1 @@
+venv*/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "transformer_models/electra-base-igc-is"]
+	path = transformer_models/electra-base-igc-is
+	url = https://huggingface.co/Icelandic-lt/electra-base-igc-is

.python-version

@@ -0,0 +1 @@
+3.8

README.md

@@ -1,3 +1,69 @@
----
-license: apache-2.0
----
+## A Universal Dependency parser built on top of a Transformer language model
+
+Score on pre-tokenized test data:
+
+```
+Metric     | Precision |    Recall |  F1 Score | AligndAcc
+-----------+-----------+-----------+-----------+-----------
+Tokens     |     99.70 |     99.77 |     99.73 |
+Sentences  |    100.00 |    100.00 |    100.00 |
+Words      |     99.62 |     99.61 |     99.61 |
+UPOS       |     96.99 |     96.97 |     96.98 |     97.36
+XPOS       |     93.65 |     93.64 |     93.65 |     94.01
+UFeats     |     91.31 |     91.29 |     91.30 |     91.65
+AllTags    |     86.86 |     86.85 |     86.86 |     87.19
+Lemmas     |     95.83 |     95.81 |     95.82 |     96.19
+UAS        |     89.01 |     89.00 |     89.00 |     89.35
+LAS        |     85.72 |     85.70 |     85.71 |     86.04
+CLAS       |     81.39 |     80.91 |     81.15 |     81.34
+MLAS       |     69.21 |     68.81 |     69.01 |     69.17
+BLEX       |     77.44 |     76.99 |     77.22 |     77.40
+```
+
+
+Score on untokenized test data:
+
+```
+Metric     | Precision |    Recall |  F1 Score | AligndAcc
+-----------+-----------+-----------+-----------+-----------
+Tokens     |     99.50 |     99.66 |     99.58 |
+Sentences  |    100.00 |    100.00 |    100.00 |
+Words      |     99.42 |     99.50 |     99.46 |
+UPOS       |     96.80 |     96.88 |     96.84 |     97.37
+XPOS       |     93.48 |     93.56 |     93.52 |     94.03
+UFeats     |     91.13 |     91.20 |     91.16 |     91.66
+AllTags    |     86.71 |     86.78 |     86.75 |     87.22
+Lemmas     |     95.66 |     95.74 |     95.70 |     96.22
+UAS        |     88.76 |     88.83 |     88.80 |     89.28
+LAS        |     85.49 |     85.55 |     85.52 |     85.99
+CLAS       |     81.19 |     80.73 |     80.96 |     81.31
+MLAS       |     69.06 |     68.67 |     68.87 |     69.16
+BLEX       |     77.28 |     76.84 |     77.06 |     77.39
+````
+
+To use the model, you need to setup COMBO, which makes it possible to use word embeddings from a pre-trained transformer model ([electra-base-igc-is](https://huggingface.co/Icelandic-lt/electra-base-igc-is)).
+
+```bash
+git submodule update --init --recursive
+pip install -U pip setuptools wheel
+pip install --index-url https://pypi.clarin-pl.eu/simple combo==1.0.5
+```
+
+* For Python 3.9, you might need to install cython:
+
+```bash
+pip install -U pip cython
+```
+
+* Then you can run the model as it is done in parse_file.py
+
+For more instructions, see here: https://gitlab.clarin-pl.eu/syntactic-tools/combo
+
+The Tokenizer directory is a clone of [Miðeind's tokenizer](https://github.com/icelandic-lt/Tokenizer).
+
+The directory `transformer_models/` contains the pretrained model [electra-base-igc-is](https://huggingface.co/Icelandic-lt/electra-base-igc-is),
+which supplies the parser with contextual embeddings and attention, trained by Jón Friðrik Daðason.
+
+
+## License
+https://opensource.org/licenses/Apache-2.0

Tokenizer

@@ -0,0 +1 @@
+Subproject commit 5ae4551ad3a3a99ad657bd0528dd4147f4f5f95f

combo-is-combined-v211/best.th

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98106212011d3b33fffa708d8497e0266bf233244b5209a6b2027640aeb729e9
+size 488929748

combo-is-combined-v211/config.json

@@ -0,0 +1,355 @@
+{
+    "dataset_reader": {
+        "type": "conllu",
+        "features": [
+            "token",
+            "char"
+        ],
+        "lemma_indexers": {
+            "char": {
+                "type": "characters_const_padding",
+                "character_tokenizer": {
+                    "end_tokens": [
+                        "__END__"
+                    ],
+                    "start_tokens": [
+                        "__START__"
+                    ]
+                },
+                "min_padding_length": 32,
+                "namespace": "lemma_characters"
+            }
+        },
+        "targets": [
+            "deprel",
+            "feats",
+            "head",
+            "lemma",
+            "upostag",
+            "xpostag"
+        ],
+        "token_indexers": {
+            "char": {
+                "type": "characters_const_padding",
+                "character_tokenizer": {
+                    "end_tokens": [
+                        "__END__"
+                    ],
+                    "start_tokens": [
+                        "__START__"
+                    ]
+                },
+                "min_padding_length": 32
+            },
+            "feats": {
+                "type": "feats_indexer"
+            },
+            "lemma": {
+                "type": "characters_const_padding",
+                "character_tokenizer": {
+                    "end_tokens": [
+                        "__END__"
+                    ],
+                    "start_tokens": [
+                        "__START__"
+                    ]
+                },
+                "min_padding_length": 32
+            },
+            "token": {
+                "type": "pretrained_transformer_mismatched_fixed",
+                "model_name": "transformer_models/electra-base-igc-is/",
+                "tokenizer_kwargs": {}
+            },
+            "upostag": {
+                "type": "single_id",
+                "feature_name": "pos_",
+                "namespace": "upostag"
+            },
+            "xpostag": {
+                "type": "single_id",
+                "feature_name": "tag_",
+                "namespace": "xpostag"
+            }
+        },
+        "use_sem": false
+    },
+    "model": {
+        "type": "semantic_multitask",
+        "dependency_relation": {
+            "type": "combo_dependency_parsing_from_vocab",
+            "dependency_projection_layer": {
+                "activation": "tanh",
+                "dropout_rate": 0.25,
+                "in_features": 1024,
+                "out_features": 128
+            },
+            "head_predictor": {
+                "cycle_loss_n": 0,
+                "dependency_projection_layer": {
+                    "activation": "tanh",
+                    "in_features": 1024,
+                    "out_features": 512
+                },
+                "head_projection_layer": {
+                    "activation": "tanh",
+                    "in_features": 1024,
+                    "out_features": 512
+                }
+            },
+            "head_projection_layer": {
+                "activation": "tanh",
+                "dropout_rate": 0.25,
+                "in_features": 1024,
+                "out_features": 128
+            },
+            "vocab_namespace": "deprel_labels"
+        },
+        "lemmatizer": {
+            "type": "combo_lemma_predictor_from_vocab",
+            "activations": [
+                "relu",
+                "relu",
+                "relu",
+                "linear"
+            ],
+            "char_vocab_namespace": "token_characters",
+            "dilation": [
+                1,
+                2,
+                4,
+                1
+            ],
+            "embedding_dim": 256,
+            "filters": [
+                256,
+                256,
+                256
+            ],
+            "input_projection_layer": {
+                "activation": "tanh",
+                "dropout_rate": 0.25,
+                "in_features": 1024,
+                "out_features": 32
+            },
+            "kernel_size": [
+                3,
+                3,
+                3,
+                1
+            ],
+            "lemma_vocab_namespace": "lemma_characters",
+            "padding": [
+                1,
+                2,
+                4,
+                0
+            ],
+            "stride": [
+                1,
+                1,
+                1,
+                1
+            ]
+        },
+        "loss_weights": {
+            "deprel": 0.8,
+            "feats": 0.2,
+            "head": 0.2,
+            "lemma": 0.05,
+            "semrel": 0.05,
+            "upostag": 0.05,
+            "xpostag": 0.05
+        },
+        "morphological_feat": {
+            "type": "combo_morpho_from_vocab",
+            "activations": [
+                "tanh",
+                "linear"
+            ],
+            "dropout": [
+                0.25,
+                0
+            ],
+            "hidden_dims": [
+                128
+            ],
+            "input_dim": 1024,
+            "num_layers": 2,
+            "vocab_namespace": "feats_labels"
+        },
+        "regularizer": {
+            "regexes": [
+                [
+                    ".*conv1d.*",
+                    {
+                        "alpha": 1e-06,
+                        "type": "l2"
+                    }
+                ],
+                [
+                    ".*forward.*",
+                    {
+                        "alpha": 1e-06,
+                        "type": "l2"
+                    }
+                ],
+                [
+                    ".*backward.*",
+                    {
+                        "alpha": 1e-06,
+                        "type": "l2"
+                    }
+                ],
+                [
+                    ".*char_embed.*",
+                    {
+                        "alpha": 1e-05,
+                        "type": "l2"
+                    }
+                ]
+            ]
+        },
+        "seq_encoder": {
+            "type": "combo_encoder",
+            "layer_dropout_probability": 0.33,
+            "stacked_bilstm": {
+                "hidden_size": 512,
+                "input_size": 164,
+                "layer_dropout_probability": 0.33,
+                "num_layers": 2,
+                "recurrent_dropout_probability": 0.33
+            }
+        },
+        "text_field_embedder": {
+            "type": "basic",
+            "token_embedders": {
+                "char": {
+                    "type": "char_embeddings_from_config",
+                    "dilated_cnn_encoder": {
+                        "activations": [
+                            "relu",
+                            "relu",
+                            "linear"
+                        ],
+                        "dilation": [
+                            1,
+                            2,
+                            4
+                        ],
+                        "filters": [
+                            512,
+                            256,
+                            64
+                        ],
+                        "input_dim": 64,
+                        "kernel_size": [
+                            3,
+                            3,
+                            3
+                        ],
+                        "padding": [
+                            1,
+                            2,
+                            4
+                        ],
+                        "stride": [
+                            1,
+                            1,
+                            1
+                        ]
+                    },
+                    "embedding_dim": 64
+                },
+                "token": {
+                    "type": "transformers_word_embeddings",
+                    "model_name": "transformer_models/electra-base-igc-is/",
+                    "projection_dim": 100
+                }
+            }
+        },
+        "upos_tagger": {
+            "activations": [
+                "tanh",
+                "linear"
+            ],
+            "dropout": [
+                0.25,
+                0
+            ],
+            "hidden_dims": [
+                64
+            ],
+            "input_dim": 1024,
+            "num_layers": 2,
+            "vocab_namespace": "upostag_labels"
+        },
+        "xpos_tagger": {
+            "activations": [
+                "tanh",
+                "linear"
+            ],
+            "dropout": [
+                0.25,
+                0
+            ],
+            "hidden_dims": [
+                128
+            ],
+            "input_dim": 1024,
+            "num_layers": 2,
+            "vocab_namespace": "xpostag_labels"
+        }
+    },
+    "train_data_path": "is_combined-ud-train.conllu",
+    "validation_data_path": "is_combined-ud-dev.conllu",
+    "trainer": {
+        "type": "gradient_descent_validate_n",
+        "checkpointer": {
+            "type": "finishing_only_checkpointer"
+        },
+        "cuda_device": 0,
+        "epoch_callbacks": [
+            {
+                "type": "transfer_patience"
+            }
+        ],
+        "grad_clipping": 5,
+        "learning_rate_scheduler": {
+            "type": "combo_scheduler"
+        },
+        "num_epochs": 400,
+        "optimizer": {
+            "type": "adam",
+            "betas": [
+                0.9,
+                0.9
+            ],
+            "lr": 0.002
+        },
+        "patience": 1,
+        "validation_metric": "+EM"
+    },
+    "vocabulary": {
+        "type": "from_instances_extended",
+        "non_padded_namespaces": [
+            "head_labels"
+        ],
+        "only_include_pretrained_words": true,
+        "oov_token": "_",
+        "padding_token": "__PAD__"
+    },
+    "type": "default",
+    "data_loader": {
+        "batch_sampler": {
+            "type": "token_count",
+            "word_batch_size": 2500
+        }
+    },
+    "datasets_for_vocab_creation": [
+        "train"
+    ],
+    "numpy_seed": 8787,
+    "pytorch_seed": 8787,
+    "random_seed": 8787
+}

combo-is-combined-v211/metrics.json

@@ -0,0 +1,81 @@
+{
+  "best_epoch": 210,
+  "peak_worker_0_memory_MB": 8525.6796875,
+  "peak_gpu_0_memory_MB": 2498.61328125,
+  "training_duration": "7:24:49.411776",
+  "training_start_epoch": 0,
+  "training_epochs": 210,
+  "epoch": 210,
+  "training_UPOS_ACC": 0.9787108946284663,
+  "training_XPOS_ACC": 0.9391129495876493,
+  "training_SEMREL_ACC": 0.0,
+  "training_LEMMA_ACC": 0.9803898255479717,
+  "training_FEATS_ACC": 0.919626898433172,
+  "training_EM": 0.8210317492485046,
+  "training_UAS": 0.94157843024532,
+  "training_LAS": 0.9163996926968465,
+  "training_UEM": 0.5165940054495913,
+  "training_LEM": 0.3965122615803815,
+  "training_EUAS": 0.0,
+  "training_ELAS": 0.0,
+  "training_EUEM": 0.0,
+  "training_ELEM": 0.0,
+  "training_partial_loss/upostag_loss": 0.1905619502067566,
+  "training_partial_loss/xpostag_loss": 0.6266801357269287,
+  "training_partial_loss/feats_loss": 1.8628418445587158,
+  "training_partial_loss/lemma_loss": 0.06094415858387947,
+  "training_partial_loss/head_loss": 0.5034704804420471,
+  "training_partial_loss/deprel_loss": 0.3214435279369354,
+  "training_partial_loss/cycle_loss": 0.0,
+  "training_loss": 0.973405952516355,
+  "training_reg_loss": 0.2107603544565408,
+  "training_worker_0_memory_MB": 8525.6796875,
+  "training_gpu_0_memory_MB": 2498.61328125,
+  "training_patience": 1,
+  "validation_UPOS_ACC": 0.9768967445412763,
+  "validation_XPOS_ACC": 0.9429237756229287,
+  "validation_SEMREL_ACC": 0.0,
+  "validation_LEMMA_ACC": 0.9705140269765286,
+  "validation_FEATS_ACC": 0.9284603739617855,
+  "validation_EM": 0.7663625478744507,
+  "validation_UAS": 0.8889570803158627,
+  "validation_LAS": 0.8563888547931754,
+  "validation_UEM": 0.3770460601446517,
+  "validation_LEM": 0.2700799390940236,
+  "validation_EUAS": 0.0,
+  "validation_ELAS": 0.0,
+  "validation_EUEM": 0.0,
+  "validation_ELEM": 0.0,
+  "validation_partial_loss/upostag_loss": 0.3415434658527374,
+  "validation_partial_loss/xpostag_loss": 0.820046067237854,
+  "validation_partial_loss/feats_loss": 1.8135327100753784,
+  "validation_partial_loss/lemma_loss": 0.12137448042631149,
+  "validation_partial_loss/head_loss": 1.9168964624404907,
+  "validation_partial_loss/deprel_loss": 25.25812530517578,
+  "validation_partial_loss/cycle_loss": 0.0,
+  "validation_loss": 24.833287269904694,
+  "validation_reg_loss": 0.0,
+  "best_validation_UPOS_ACC": 0.9768967445412763,
+  "best_validation_XPOS_ACC": 0.9429237756229287,
+  "best_validation_SEMREL_ACC": 0.0,
+  "best_validation_LEMMA_ACC": 0.9705140269765286,
+  "best_validation_FEATS_ACC": 0.9284603739617855,
+  "best_validation_EM": 0.7663625478744507,
+  "best_validation_UAS": 0.8889570803158627,
+  "best_validation_LAS": 0.8563888547931754,
+  "best_validation_UEM": 0.3770460601446517,
+  "best_validation_LEM": 0.2700799390940236,
+  "best_validation_EUAS": 0.0,
+  "best_validation_ELAS": 0.0,
+  "best_validation_EUEM": 0.0,
+  "best_validation_ELEM": 0.0,
+  "best_validation_partial_loss/upostag_loss": 0.3415434658527374,
+  "best_validation_partial_loss/xpostag_loss": 0.820046067237854,
+  "best_validation_partial_loss/feats_loss": 1.8135327100753784,
+  "best_validation_partial_loss/lemma_loss": 0.12137448042631149,
+  "best_validation_partial_loss/head_loss": 1.9168964624404907,
+  "best_validation_partial_loss/deprel_loss": 25.25812530517578,
+  "best_validation_partial_loss/cycle_loss": 0.0,
+  "best_validation_loss": 24.833287269904694,
+  "best_validation_reg_loss": 0.0
+}

combo-is-combined-v211/vocabulary/deprel_labels.txt

@@ -0,0 +1,37 @@
+_
+punct
+obl
+nsubj
+case
+advmod
+cc
+conj
+mark
+root
+obj
+amod
+nmod:poss
+cop
+ccomp
+aux
+det
+advcl
+acl
+acl:relcl
+compound:prt
+appos
+xcomp
+iobj
+dep
+flat:name
+nummod
+parataxis
+vocative
+nmod
+expl
+flat:foreign
+discourse
+fixed
+csubj
+compound
+dislocated

combo-is-combined-v211/vocabulary/feats_labels.txt

@@ -0,0 +1,50 @@
+_
+Number=Sing
+Definite=Ind
+Gender=Masc
+Case=Nom
+Voice=Act
+VerbForm=Fin
+Number=Plur
+Mood=Ind
+Gender=Neut
+Person=3
+PronType=Prs
+Case=Acc
+Case=Dat
+Gender=Fem
+Tense=Past
+Tense=Pres
+Case=Gen
+Definite=Def
+Degree=Pos
+Person=1
+VerbForm=Inf
+PronType=Dem
+PronType=Ind
+Person=2
+Mood=Sub
+Voice=Mid
+VerbForm=Part
+VerbForm=Sup
+Degree=Cmp
+NumType=Card
+Degree=Sup
+Foreign=Yes
+PronType=Int
+Mood=Imp
+NumType=Ord
+NumType=Frac
+Number=None
+Definite=None
+Gender=None
+Case=None
+Voice=None
+VerbForm=None
+Mood=None
+Person=None
+PronType=None
+Tense=None
+Degree=None
+NumType=None
+Foreign=None

combo-is-combined-v211/vocabulary/lemma_characters.txt

@@ -0,0 +1,94 @@
+_
+__START__
+__END__
+a
+r
+n
+e
+i
+g
+s
+l
+u
+ð
+t
+k
+h
+m
+v
+f
+o
+þ
+á
+j
+d
+í
+.
+,
+ú
+ó
+b
+y
+p
+é
+æ
+ö
+"
+ý
+:
+?
+;
+x
+c
+1
+0
+2
+!
+z
+6
+5
+-
+3
+4
+9
+7
+8
+w
+„
+“
+q
+S
+—
+R
+$
+Þ
+M
+/
+G
+L
+B
+Ó
+H
+å
+A
+E
+N
+ü
+ä
+U
+P
+K
+|
+Å
+I
+Ð
+V
+F
+O
+T
+Ú
+Ö
+Á
+Í
+<
+>

combo-is-combined-v211/vocabulary/non_padded_namespaces.txt

@@ -0,0 +1 @@
+head_labels

combo-is-combined-v211/vocabulary/token_characters.txt

@@ -0,0 +1,108 @@
+_
+__START__
+__END__
+a
+r
+n
+i
+e
+s
+u
+t
+ð
+g
+l
+m
+k
+o
+f
+v
+h
+þ
+á
+d
+í
+.
+,
+ó
+j
+b
+y
+ö
+p
+æ
+ú
+é
+Þ
+"
+H
+E
+S
+ý
+G
+:
+V
+A
+K
+J
+M
+O
+B
+F
+N
+?
+;
+D
+x
+L
+R
+Á
+É
+T
+P
+I
+c
+1
+Ó
+Í
+0
+2
+U
+z
+!
+6
+-
+Ö
+5
+3
+4
+9
+8
+7
+C
+Æ
+Ú
+X
+Y
+W
+„
+“
+/
+<
+>
+Ð
+q
+w
+Ý
+—
+Z
+å
+Q
+ü
+ä
+|
++
+Å
+$
+&
+'

combo-is-combined-v211/vocabulary/upostag_labels.txt

@@ -0,0 +1,17 @@
+_
+NOUN
+VERB
+PRON
+PUNCT
+ADP
+ADV
+CCONJ
+AUX
+DET
+PROPN
+SCONJ
+ADJ
+PART
+NUM
+X
+INTJ

combo-is-combined-v211/vocabulary/xpostag_labels.txt

@@ -0,0 +1,280 @@
+_
+P
+.
+CONJ
+ADV
+PRO-N
+VBDI
+,
+N-A
+C
+N-D
+N-N
+PRO-D
+VBPI
+VB
+NPR-N
+PRO-A
+ADJ-N
+BEDI
+N-G
+NS-A
+RP
+TO
+"
+NS-D
+PRO-G
+BEPI
+NS-N
+NEG
+VBN
+ADVR
+VAN
+Q-N
+NPR-D
+ADJ-A
+NPR-G
+Q-A
+D-N
+D-A
+MDPI
+ADJ-D
+NS-G
+NPR-A
+D-D
+Q-D
+HVDI
+HVPI
+VBPS
+MDDS
+VBDS
+FW
+WADV
+NUM-A
+ALSO
+ADJ-G
+BE
+BEDS
+MDDI
+NUM-N
+WPRO-N
+VBI
+ADVS
+RDDI
+ES
+D-G
+Q-G
+BEPS
+HV
+ADJS-N
+HVDS
+BEN
+ONE-N
+INTJ
+ADJR-N
+VAG
+ONE-A
+RDPI
+OTHER-A
+DO
+DODI
+MDPS
+NPRS-D
+OTHER-N
+MD
+NPRS-N
+OTHER-D
+ONE-D
+FP
+WPRO-A
+HVPS
+NUM-D
+QR-N
+WQ
+ADJS-A
+RDN
+QR-A
+RD
+DOPI
+NUM-G
+RPX
+VAN-A
+ADJR-A
+ADJS-D
+QS-N
+SUCH-A
+OTHER-G
+NPRS-G
+VAN-D
+DON
+ADJR-D
+NPRS-A
+DAN
+RDPS
+RDDS
+WPRO-D
+OTHERS-N
+OTHERS-D
+SUCH-N
+WD-N
+SUCH-D
+QS-A
+QS-D
+HVN
+QR-D
+WPRO-G
+OTHERS-A
+WD-A
+VAG-A
+MDN
+ADJR-G
+VAG-D
+FP-N
+CONJ-1
+OTHERS-G
+WD-D
+ONE-G
+BEI
+ADJS-G
+Q
+DOPS
+HAN
+NUM
+MAN-N
+SUCH-G
+N
+FP-A
+WQ-N
+DOI
+QR-G
+WPRO
+CONJ-2
+QS
+VAN-G
+HVI
+VAG-G
+FP-D
+ADJ
+QS-G
+QR
+DODS
+X
+SUCH
+VBN-D
+LS
+CONJ-3
+WQ-A
+ADJR
+CONJ-4
+WD-G
+VBN-A
+VAN-N
+BAN
+VAG-N
+DAN-A
+NPR
+VB-1
+DAN-D
+WQ-D
+NS
+DAG
+WQ-G
+:
+HAG
+OTHER
+WPRO-1
+MAN-D
+as
+ADJS
+;
+VBN-G
+WADJ-N
+MAG
+RX
+PRO
+RP-2
+CONJ-5
+MDI
+LB
+ONES-N
+WADV-N
+VB-3
+MAN-A
+RP-3
+RDI
+FP-G
+WADV-D
+NEG-1
+FW-D
+G
+VB-2
+TP
+ONE
+MANS-N
+NP
+VAN-A-4
+CONJ-8
+POR-D
+CONJ-6
+RPO-D
+CONJ-9
+VBDP
+NUM-1
+WPRO-N-1
+DVBN
+NPR-1
+MS-N
+POR-A
+WRPO-N
+QDJ-A
+VAN-4
+DV
+BAG
+Q-ADV
+VPDI
+ONES-G
+NPR-V
+ADV-N
+PRO-N-YYY
+REP
+ADJP
+FP-1
+NEG-2
+WADVP-10
+FOREIGN
+WADVP-5
+RP-6
+CONJ-7
+WDD
+IP-INF
+WADV-A
+ADJP-N
+M-D
+ADVS-N
+ONES-A
+FP-N-3
+RAN
+DW
+ADV-1
+VBPI$
+ADVR-A
+QADV
+VDN
+RD-3
+ADVR-D
+ES-A
+PRON
+OTHER-WPRO
+MAG-A
+WS-A
+AN
+RP-1
+ES-1
+PP
+RAG
+ALDJ-N
+FP-2
+N=G
+WD
+VB-4
+HV-1

parse_file.py

@@ -0,0 +1,35 @@
+from combo.predict import COMBO
+from allennlp.data import tokenizers
+from argparse import ArgumentParser
+
+parser = ArgumentParser()
+parser.add_argument('--parser')
+parser.add_argument('--infile')
+parser.add_argument('--pretokenized', action='store_true')
+args = parser.parse_args()
+
+# If your data is pre-tokenized, you can add the --pretokenized flag
+# If you have a GPU available, you can add cuda_device=<your-device> to COMBO.from_pretrained
+# The parser expects input in the same format as test_file.txt, i.e. one sentence per line
+
+if args.pretokenized:
+    from Tokenizer.src.tokenizer import split_into_sentences
+    nlp = COMBO.from_pretrained('combo-is-combined-v211', tokenizer=tokenizers.SpacyTokenizer(split_on_spaces=True))
+else:
+    nlp = COMBO.from_pretrained(args.parser)
+
+def read_test_file(file):
+    with open(file, 'r', encoding='utf-8') as infile:
+        for line in infile:
+            if args.pretokenized:
+                yield ' '.join(split_into_sentences(line))
+            else:
+                yield line.rstrip()
+
+test_file = read_test_file(args.infile)
+
+for sent in test_file:
+    sentence = nlp(sent)
+    for index, token in enumerate(sentence.tokens, 1):
+       print(f'{token.id}\t{token.token}\t{token.lemma}\t{token.upostag}\t{token.xpostag}\t{token.feats}\t{token.head}\t{token.deprel}\t{token.deps}\t{token.misc}')
+    print()

requirements.txt

@@ -0,0 +1,4 @@
+allennlp==2.10.1
+combo==0.1.3
+diaparser==1.1.2
+Tokenizer==3.4.2

test_file.txt

@@ -0,0 +1,3 @@
+Komið þið sæl.
+Þetta skjal er ætlað til að sýna hvernig þáttarinn virkar.
+Njótið dagsins.

transformer_models/electra-base-igc-is

@@ -0,0 +1 @@
+Subproject commit e2921de06b441e2a3066da485d6fa31cf5c816a8