Upload folder using huggingface_hub

Basque/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 42
+}

Basque/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Basque/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce322b9b6b1180c639fcb02a177395ce5a1bcde446a9dbe983a98dfe3821ca93
+size 2536728

Basque/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Basque/tokenizer.json

@@ -0,0 +1,159 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "b": 4,
+      "ai̯": 5,
+      "e": 6,
+      "s̪̻": 7,
+      "ɟ": 8,
+      "ei̯": 9,
+      "t̺s̺": 10,
+      "i": 11,
+      "oi̯": 12,
+      "a": 13,
+      "ɾ": 14,
+      "k": 15,
+      "t̠ʃ": 16,
+      "s̺": 17,
+      "l": 18,
+      "p": 19,
+      "o": 20,
+      "r": 21,
+      "t̪": 22,
+      "u": 23,
+      "n": 24,
+      "m": 25,
+      "ð": 26,
+      "t̪̻s̪̻": 27,
+      "β": 28,
+      "ʎ": 29,
+      "ɡ": 30,
+      "ɣ": 31,
+      "au̯": 32,
+      "c": 33,
+      "j": 34,
+      "d̪": 35,
+      "ʃ": 36,
+      "ɲ": 37,
+      "f": 38,
+      "eu̯": 39,
+      "θ": 40,
+      "x": 41
+    },
+    "unk_token": "UNK"
+  }
+}

Basque/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Basque/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:743e25492f2a0dd2aac1435f2a19a6367f3618135945a12fa336ec4c809cb4e8
+size 5368

Basque/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"b":4,"ai̯":5,"e":6,"s̪̻":7,"ɟ":8,"ei̯":9,"t̺s̺":10,"i":11,"oi̯":12,"a":13,"ɾ":14,"k":15,"t̠ʃ":16,"s̺":17,"l":18,"p":19,"o":20,"r":21,"t̪":22,"u":23,"n":24,"m":25,"ð":26,"t̪̻s̪̻":27,"β":28,"ʎ":29,"ɡ":30,"ɣ":31,"au̯":32,"c":33,"j":34,"d̪":35,"ʃ":36,"ɲ":37,"f":38,"eu̯":39,"θ":40,"x":41}

Cantonese/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 152
+}

Cantonese/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Cantonese/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b48efa4f503665cc6436b384127aac0bef14d1cc791bd59884828eb52fe72b57
+size 2593048

Cantonese/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Cantonese/tokenizer.json

@@ -0,0 +1,269 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "WhitespaceSplit"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "j": 4,
+      "ɐ˥": 5,
+      "t": 6,
+      "k": 7,
+      "ɐu˧˥": 8,
+      "i˨": 9,
+      "n": 10,
+      "i˧˩̰": 11,
+      "y˨": 12,
+      "s": 13,
+      "ɐ˨": 14,
+      "p": 15,
+      "ts": 16,
+      "ɐu˥": 17,
+      "ɪ̞˧˥": 18,
+      "ŋ": 19,
+      "ɵ˧": 20,
+      "a̞˧": 21,
+      "l": 22,
+      "ʊ̟˥": 23,
+      "a̞˧˩̰": 24,
+      "ɛ˥": 25,
+      "ei˩˧": 26,
+      "w": 27,
+      "a̞˨": 28,
+      "ɐi˧˥": 29,
+      "a̞˧˥": 30,
+      "m̩˧˥": 31,
+      "m": 32,
+      "ou˥": 33,
+      "ei˥": 34,
+      "i˧": 35,
+      "ɔ̽˧˥": 36,
+      "tʰ": 37,
+      "i˥": 38,
+      "f": 39,
+      "aːĭ˧": 40,
+      "h": 41,
+      "ɵy˧": 42,
+      "a̞˥": 43,
+      "ei˧˩̰": 44,
+      "ou˨": 45,
+      "ɔ̽˧": 46,
+      "ɐi˧˩̰": 47,
+      "u˧": 48,
+      "ɔːĭ˥": 49,
+      "ɐu˨": 50,
+      "ei˧˥": 51,
+      "ɐi˨": 52,
+      "ʊ̟˧˩̰": 53,
+      "ʊ̟˨": 54,
+      "a̞˩˧": 55,
+      "ou˧˥": 56,
+      "aːĭ˧˥": 57,
+      "ɔ̽˨": 58,
+      "ɛ˩˧": 59,
+      "ɪ̞˨": 60,
+      "iːŭ˧": 61,
+      "ɛ˧˩̰": 62,
+      "m̩˧˩̰": 63,
+      "ɵ˧˥": 64,
+      "ei˧": 65,
+      "ɐu˧˩̰": 66,
+      "m̩˧": 67,
+      "ɐ˧˥": 68,
+      "ɐu˩˧": 69,
+      "ɐi˥": 70,
+      "ɔ̽˥": 71,
+      "ɔ̽˧˩̰": 72,
+      "ɔːĭ˧": 73,
+      "ou˩˧": 74,
+      "m̩˥": 75,
+      "ɐ˧": 76,
+      "tsʰ": 77,
+      "ɛ˧˥": 78,
+      "i˧˥": 79,
+      "ɔ̽˩˧": 80,
+      "kʰ": 81,
+      "ɐ˧˩̰": 82,
+      "aːŭ˧˥": 83,
+      "pʰ": 84,
+      "aːĭ˧˩̰": 85,
+      "ɵy˩˧": 86,
+      "ɛ˧": 87,
+      "u˧˥": 88,
+      "ɛ˨": 89,
+      "ʊ̟˧": 90,
+      "u˥": 91,
+      "m̩˩˧": 92,
+      "aːŭ˧": 93,
+      "œ̞˩˧": 94,
+      "i˩˧": 95,
+      "ɪ̞˧˩̰": 96,
+      "u˨": 97,
+      "ɪ̞˥": 98,
+      "iːŭ˧˩̰": 99,
+      "œ̞˧˥": 100,
+      "y˧": 101,
+      "uːĭ˩˧": 102,
+      "uːĭ˥": 103,
+      "ɵy˧˥": 104,
+      "y˧˩̰": 105,
+      "ɔːĭ˧˥": 106,
+      "ɛ": 107,
+      "ou˧": 108,
+      "ei˨": 109,
+      "ɵ˥": 110,
+      "u˧˩̰": 111,
+      "y˥": 112,
+      "œ̞˥": 113,
+      "œ̞˧˩̰": 114,
+      "aːĭ˨": 115,
+      "ɐ˩˧": 116,
+      "œ̞˧": 117,
+      "uːĭ˧˥": 118,
+      "ɐu˧": 119,
+      "ɐi˩˧": 120,
+      "ɐi˧": 121,
+      "ou˧˩̰": 122,
+      "aːĭ˥": 123,
+      "aːŭ˥": 124,
+      "ŋ˩˧": 125,
+      "y˧˥": 126,
+      "iːŭ˥": 127,
+      "ɔːĭ˨": 128,
+      "ʊ̟˧˥": 129,
+      "iːŭ˧˥": 130,
+      "ɵy˥": 131,
+      "ɔːĭ˧˩̰": 132,
+      "uːĭ˧": 133,
+      "ɵy˧˩̰": 134,
+      "œ̞˨": 135,
+      "m̩˨": 136,
+      "aːŭ˧˩̰": 137,
+      "y˩˧": 138,
+      "aːŭ˩˧": 139,
+      "aːĭ˩˧": 140,
+      "uːĭ˨": 141,
+      "ɵy˨": 142,
+      "aːŭ˨": 143,
+      "ɪ̞˧": 144,
+      "ɵ˨": 145,
+      "iːŭ˩˧": 146,
+      "iːŭ˨": 147,
+      "ɵ˧˩̰": 148,
+      "uːĭ˧˩̰": 149,
+      "u˩˧": 150,
+      "ŋ˧˩̰": 151
+    },
+    "unk_token": "UNK"
+  }
+}

Cantonese/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Cantonese/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92167da0e935ca89f7238c7dc0210aedb733dd0e02ec79d330b22ab9ee613d4
+size 5368

Cantonese/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"j":4,"ɐ˥":5,"t":6,"k":7,"ɐu˧˥":8,"i˨":9,"n":10,"i˧˩̰":11,"y˨":12,"s":13,"ɐ˨":14,"p":15,"ts":16,"ɐu˥":17,"ɪ̞˧˥":18,"ŋ":19,"ɵ˧":20,"a̞˧":21,"l":22,"ʊ̟˥":23,"a̞˧˩̰":24,"ɛ˥":25,"ei˩˧":26,"w":27,"a̞˨":28,"ɐi˧˥":29,"a̞˧˥":30,"m̩˧˥":31,"m":32,"ou˥":33,"ei˥":34,"i˧":35,"ɔ̽˧˥":36,"tʰ":37,"i˥":38,"f":39,"aːĭ˧":40,"h":41,"ɵy˧":42,"a̞˥":43,"ei˧˩̰":44,"ou˨":45,"ɔ̽˧":46,"ɐi˧˩̰":47,"u˧":48,"ɔːĭ˥":49,"ɐu˨":50,"ei˧˥":51,"ɐi˨":52,"ʊ̟˧˩̰":53,"ʊ̟˨":54,"a̞˩˧":55,"ou˧˥":56,"aːĭ˧˥":57,"ɔ̽˨":58,"ɛ˩˧":59,"ɪ̞˨":60,"iːŭ˧":61,"ɛ˧˩̰":62,"m̩˧˩̰":63,"ɵ˧˥":64,"ei˧":65,"ɐu˧˩̰":66,"m̩˧":67,"ɐ˧˥":68,"ɐu˩˧":69,"ɐi˥":70,"ɔ̽˥":71,"ɔ̽˧˩̰":72,"ɔːĭ˧":73,"ou˩˧":74,"m̩˥":75,"ɐ˧":76,"tsʰ":77,"ɛ˧˥":78,"i˧˥":79,"ɔ̽˩˧":80,"kʰ":81,"ɐ˧˩̰":82,"aːŭ˧˥":83,"pʰ":84,"aːĭ˧˩̰":85,"ɵy˩˧":86,"ɛ˧":87,"u˧˥":88,"ɛ˨":89,"ʊ̟˧":90,"u˥":91,"m̩˩˧":92,"aːŭ˧":93,"œ̞˩˧":94,"i˩˧":95,"ɪ̞˧˩̰":96,"u˨":97,"ɪ̞˥":98,"iːŭ˧˩̰":99,"œ̞˧˥":100,"y˧":101,"uːĭ˩˧":102,"uːĭ˥":103,"ɵy˧˥":104,"y˧˩̰":105,"ɔːĭ˧˥":106,"ɛ":107,"ou˧":108,"ei˨":109,"ɵ˥":110,"u˧˩̰":111,"y˥":112,"œ̞˥":113,"œ̞˧˩̰":114,"aːĭ˨":115,"ɐ˩˧":116,"œ̞˧":117,"uːĭ˧˥":118,"ɐu˧":119,"ɐi˩˧":120,"ɐi˧":121,"ou˧˩̰":122,"aːĭ˥":123,"aːŭ˥":124,"ŋ˩˧":125,"y˧˥":126,"iːŭ˥":127,"ɔːĭ˨":128,"ʊ̟˧˥":129,"iːŭ˧˥":130,"ɵy˥":131,"ɔːĭ˧˩̰":132,"uːĭ˧":133,"ɵy˧˩̰":134,"œ̞˨":135,"m̩˨":136,"aːŭ˧˩̰":137,"y˩˧":138,"aːŭ˩˧":139,"aːĭ˩˧":140,"uːĭ˨":141,"ɵy˨":142,"aːŭ˨":143,"ɪ̞˧":144,"ɵ˨":145,"iːŭ˩˧":146,"iːŭ˨":147,"ɵ˧˩̰":148,"uːĭ˧˩̰":149,"u˩˧":150,"ŋ˧˩̰":151}

Catalan/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 40
+}

Catalan/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Catalan/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdfc9c6f2507119bf6a15f3d6a7b45bf7adfa72e1b40d075f112cd19fd6f449e
+size 2535704

Catalan/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Catalan/tokenizer.json

@@ -0,0 +1,157 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "a": 4,
+      "w": 5,
+      "ɛ": 6,
+      "ə": 7,
+      "ð": 8,
+      "t̪": 9,
+      "j": 10,
+      "i": 11,
+      "ɔ": 12,
+      "n̺": 13,
+      "z̺": 14,
+      "d̪": 15,
+      "s̺": 16,
+      "β": 17,
+      "m": 18,
+      "e": 19,
+      "f": 20,
+      "ɾ̺": 21,
+      "r̺": 22,
+      "u̯": 23,
+      "k": 24,
+      "u": 25,
+      "b": 26,
+      "p": 27,
+      "ɣ": 28,
+      "ɡ": 29,
+      "ŋ": 30,
+      "o": 31,
+      "ɫ̺": 32,
+      "ɲ̟": 33,
+      "ʒ": 34,
+      "ʃ": 35,
+      "ʎ̟": 36,
+      "t̠ʃ": 37,
+      "d̠ʒ": 38,
+      "ts̺": 39
+    },
+    "unk_token": "UNK"
+  }
+}

Catalan/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Catalan/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3a7040f7ec89ba614817a534ae93366548bd38a7b2b5590c13fa574346d1c17
+size 5368

Catalan/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"a":4,"w":5,"ɛ":6,"ə":7,"ð":8,"t̪":9,"j":10,"i":11,"ɔ":12,"n̺":13,"z̺":14,"d̪":15,"s̺":16,"β":17,"m":18,"e":19,"f":20,"ɾ̺":21,"r̺":22,"u̯":23,"k":24,"u":25,"b":26,"p":27,"ɣ":28,"ɡ":29,"ŋ":30,"o":31,"ɫ̺":32,"ɲ̟":33,"ʒ":34,"ʃ":35,"ʎ̟":36,"t̠ʃ":37,"d̠ʒ":38,"ts̺":39}

Croatian/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 35
+}

Croatian/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Croatian/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0b9958f82c31f0e4c77719708ab94a772c58997d927716c023ee288b52869a
+size 2533144

Croatian/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Croatian/tokenizer.json

@@ -0,0 +1,152 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "e": 4,
+      "a": 5,
+      "u": 6,
+      "x": 7,
+      "k": 8,
+      "t̪": 9,
+      "n": 10,
+      "o": 11,
+      "d̪": 12,
+      "i": 13,
+      "r": 14,
+      "m": 15,
+      "ʃ": 16,
+      "p": 17,
+      "s": 18,
+      "ʋ": 19,
+      "j": 20,
+      "t̠ʃ": 21,
+      "l": 22,
+      "ɡ": 23,
+      "ʒ": 24,
+      "b": 25,
+      "t̪s": 26,
+      "z": 27,
+      "d̠ʒ": 28,
+      "ʎ": 29,
+      "f": 30,
+      "ɲ": 31,
+      "y": 32,
+      "q": 33,
+      "w": 34
+    },
+    "unk_token": "UNK"
+  }
+}

Croatian/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Croatian/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbd3f3a66db05b16ce892b0eaac9d0e8dd0d693f3f5c25936e5ac1b8c9dc2a52
+size 5368

Croatian/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"e":4,"a":5,"u":6,"x":7,"k":8,"t̪":9,"n":10,"o":11,"d̪":12,"i":13,"r":14,"m":15,"ʃ":16,"p":17,"s":18,"ʋ":19,"j":20,"t̠ʃ":21,"l":22,"ɡ":23,"ʒ":24,"b":25,"t̪s":26,"z":27,"d̠ʒ":28,"ʎ":29,"f":30,"ɲ":31,"y":32,"q":33,"w":34}

Danish/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 55
+}

Danish/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Danish/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b4099c3bb9b42c44e7c514877643d08a7244e54e9e5e09f72ffaa87ac9da850
+size 2543384

Danish/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Danish/tokenizer.json

@@ -0,0 +1,172 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "n": 4,
+      "oˤ": 5,
+      "t": 6,
+      "y": 7,
+      "ə": 8,
+      "ð": 9,
+      "ʁ": 10,
+      "ɑˤː": 11,
+      "s": 12,
+      "k": 13,
+      "i": 14,
+      "b": 15,
+      "eˤ": 16,
+      "t̠ʃ": 17,
+      "a": 18,
+      "l": 19,
+      "d": 20,
+      "ɡ": 21,
+      "f": 22,
+      "e": 23,
+      "ɛ": 24,
+      "r": 25,
+      "ɔ": 26,
+      "w": 27,
+      "ɔˤ": 28,
+      "m": 29,
+      "uˤ": 30,
+      "j": 31,
+      "ɑ": 32,
+      "u": 33,
+      "ɒ": 34,
+      "iˤ": 35,
+      "ʋ": 36,
+      "h": 37,
+      "œ": 38,
+      "p": 39,
+      "ɕ": 40,
+      "o": 41,
+      "ŋ": 42,
+      "ɒː": 43,
+      "aˤ": 44,
+      "ɜ": 45,
+      "œː": 46,
+      "eː": 47,
+      "aː": 48,
+      "d̠ʒ": 49,
+      "uː": 50,
+      "ɔː": 51,
+      "oː": 52,
+      "iː": 53,
+      "yː": 54
+    },
+    "unk_token": "UNK"
+  }
+}

Danish/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Danish/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd62a528ea96d86ad343aab5d79a6f3e218353244a0be4c92de82947b82c1fe9
+size 5368

Danish/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"n":4,"oˤ":5,"t":6,"y":7,"ə":8,"ð":9,"ʁ":10,"ɑˤː":11,"s":12,"k":13,"i":14,"b":15,"eˤ":16,"t̠ʃ":17,"a":18,"l":19,"d":20,"ɡ":21,"f":22,"e":23,"ɛ":24,"r":25,"ɔ":26,"w":27,"ɔˤ":28,"m":29,"uˤ":30,"j":31,"ɑ":32,"u":33,"ɒ":34,"iˤ":35,"ʋ":36,"h":37,"œ":38,"p":39,"ɕ":40,"o":41,"ŋ":42,"ɒː":43,"aˤ":44,"ɜ":45,"œː":46,"eː":47,"aː":48,"d̠ʒ":49,"uː":50,"ɔː":51,"oː":52,"iː":53,"yː":54}

Dutch/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 50
+}

Dutch/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Dutch/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2419c1d9f086c904f711146ca854e725f4afa8b229953ec297133263c26d1b8d
+size 2540824

Dutch/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Dutch/tokenizer.json

@@ -0,0 +1,167 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "z": 4,
+      "oː": 5,
+      "j": 6,
+      "ãː": 7,
+      "ɦ": 8,
+      "ɾ": 9,
+      "d": 10,
+      "i": 11,
+      "ɛ": 12,
+      "p": 13,
+      "ɪ": 14,
+      "k": 15,
+      "ɑ": 16,
+      "l": 17,
+      "ɛː": 18,
+      "n": 19,
+      "s": 20,
+      "v": 21,
+      "ə": 22,
+      "ɛi": 23,
+      "ʋ": 24,
+      "t": 25,
+      "m": 26,
+      "ɣ": 27,
+      "ʏ": 28,
+      "ɔ": 29,
+      "x": 30,
+      "u": 31,
+      "f": 32,
+      "ŋ": 33,
+      "øː": 34,
+      "b": 35,
+      "ɔː": 36,
+      "ʌu": 37,
+      "y": 38,
+      "œy": 39,
+      "tʲ": 40,
+      "w": 41,
+      "ʃ": 42,
+      "t̠ʃ": 43,
+      "ɲ": 44,
+      "ʒ": 45,
+      "iː": 46,
+      "ɡ": 47,
+      "d̠ʒ": 48,
+      "ã": 49
+    },
+    "unk_token": "UNK"
+  }
+}

Dutch/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Dutch/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:051d8089b9b2a56e1beb0ffb7cb9d9effdd39e932954aebd1b6feb1c099a56c7
+size 5368

Dutch/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"z":4,"oː":5,"j":6,"ãː":7,"ɦ":8,"ɾ":9,"d":10,"i":11,"ɛ":12,"p":13,"ɪ":14,"k":15,"ɑ":16,"l":17,"ɛː":18,"n":19,"s":20,"v":21,"ə":22,"ɛi":23,"ʋ":24,"t":25,"m":26,"ɣ":27,"ʏ":28,"ɔ":29,"x":30,"u":31,"f":32,"ŋ":33,"øː":34,"b":35,"ɔː":36,"ʌu":37,"y":38,"œy":39,"tʲ":40,"w":41,"ʃ":42,"t̠ʃ":43,"ɲ":44,"ʒ":45,"iː":46,"ɡ":47,"d̠ʒ":48,"ã":49}

EnglishNA/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 128,
+  "n_head": 4,
+  "n_inner": 512,
+  "n_layer": 3,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 47
+}

EnglishNA/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}