Upload folder using huggingface_hub

Dutch/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 50
+}

Dutch/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Dutch/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb382725a383237baa18d293bec85426226eed75c349ecf65f51c75903cf475f
+size 19277120

Dutch/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Dutch/tokenizer.json

@@ -0,0 +1,167 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "z": 4,
+      "oː": 5,
+      "j": 6,
+      "ãː": 7,
+      "ɦ": 8,
+      "ɾ": 9,
+      "d": 10,
+      "i": 11,
+      "ɛ": 12,
+      "p": 13,
+      "ɪ": 14,
+      "k": 15,
+      "ɑ": 16,
+      "l": 17,
+      "ɛː": 18,
+      "n": 19,
+      "s": 20,
+      "v": 21,
+      "ə": 22,
+      "ɛi": 23,
+      "ʋ": 24,
+      "t": 25,
+      "m": 26,
+      "ɣ": 27,
+      "ʏ": 28,
+      "ɔ": 29,
+      "x": 30,
+      "u": 31,
+      "f": 32,
+      "ŋ": 33,
+      "øː": 34,
+      "b": 35,
+      "ɔː": 36,
+      "ʌu": 37,
+      "y": 38,
+      "œy": 39,
+      "tʲ": 40,
+      "w": 41,
+      "ʃ": 42,
+      "t̠ʃ": 43,
+      "ɲ": 44,
+      "ʒ": 45,
+      "iː": 46,
+      "ɡ": 47,
+      "d̠ʒ": 48,
+      "ã": 49
+    },
+    "unk_token": "UNK"
+  }
+}

Dutch/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Dutch/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95b23dfd8b5f428213743e99e188bc85395999750c9f8931fbff5727426dbce0
+size 5368

Dutch/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"z":4,"oː":5,"j":6,"ãː":7,"ɦ":8,"ɾ":9,"d":10,"i":11,"ɛ":12,"p":13,"ɪ":14,"k":15,"ɑ":16,"l":17,"ɛː":18,"n":19,"s":20,"v":21,"ə":22,"ɛi":23,"ʋ":24,"t":25,"m":26,"ɣ":27,"ʏ":28,"ɔ":29,"x":30,"u":31,"f":32,"ŋ":33,"øː":34,"b":35,"ɔː":36,"ʌu":37,"y":38,"œy":39,"tʲ":40,"w":41,"ʃ":42,"t̠ʃ":43,"ɲ":44,"ʒ":45,"iː":46,"ɡ":47,"d̠ʒ":48,"ã":49}

English/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 47
+}

English/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

English/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c54b765e3060afc585e5502181ae99612b08693df1ac0f0601d50c91a3830cee
+size 19274048

English/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

English/tokenizer.json

@@ -0,0 +1,164 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "d̠ʒ": 4,
+      "ʌ": 5,
+      "s": 6,
+      "t": 7,
+      "l": 8,
+      "aɪ": 9,
+      "k": 10,
+      "j": 11,
+      "ʊ": 12,
+      "ɹ": 13,
+      "b": 14,
+      "æ": 15,
+      "h": 16,
+      "oʊ": 17,
+      "m": 18,
+      "iː": 19,
+      "ð": 20,
+      "ɛ": 21,
+      "z": 22,
+      "f": 23,
+      "eɪ": 24,
+      "w": 25,
+      "ɪ": 26,
+      "ɡ": 27,
+      "ɑ": 28,
+      "ə": 29,
+      "p": 30,
+      "uː": 31,
+      "i": 32,
+      "θ": 33,
+      "ŋ": 34,
+      "ɔ": 35,
+      "ɔɪ": 36,
+      "n": 37,
+      "d": 38,
+      "aʊ": 39,
+      "v": 40,
+      "ɜː": 41,
+      "t̠ʃ": 42,
+      "ʃ": 43,
+      "iə": 44,
+      "ʒ": 45,
+      "x": 46
+    },
+    "unk_token": "UNK"
+  }
+}

English/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

English/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c012cec2d392656f3d6a16f802c59238740106afe16b1e7b2ed4cd1693c552da
+size 5368

English/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"d̠ʒ":4,"ʌ":5,"s":6,"t":7,"l":8,"aɪ":9,"k":10,"j":11,"ʊ":12,"ɹ":13,"b":14,"æ":15,"h":16,"oʊ":17,"m":18,"iː":19,"ð":20,"ɛ":21,"z":22,"f":23,"eɪ":24,"w":25,"ɪ":26,"ɡ":27,"ɑ":28,"ə":29,"p":30,"uː":31,"i":32,"θ":33,"ŋ":34,"ɔ":35,"ɔɪ":36,"n":37,"d":38,"aʊ":39,"v":40,"ɜː":41,"t̠ʃ":42,"ʃ":43,"iə":44,"ʒ":45,"x":46}

EnglishUK/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 51
+}

EnglishUK/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

EnglishUK/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7fd1d94d4cdf2ed713efb0d5efca1700bbea2090d49515025d0f20a92bb3e0
+size 19278144

EnglishUK/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

EnglishUK/tokenizer.json

@@ -0,0 +1,168 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "ð": 4,
+      "æ": 5,
+      "tʰ": 6,
+      "ɡ": 7,
+      "ʊ": 8,
+      "d": 9,
+      "ɑː": 10,
+      "l": 11,
+      "ɪ": 12,
+      "n": 13,
+      "eɪ": 14,
+      "t̠ʃ": 15,
+      "w": 16,
+      "ɒ": 17,
+      "ʌ": 18,
+      "z": 19,
+      "m": 20,
+      "iː": 21,
+      "aɪ": 22,
+      "h": 23,
+      "e": 24,
+      "kʰ": 25,
+      "s": 26,
+      "ə": 27,
+      "ɔː": 28,
+      "ɹ": 29,
+      "i": 30,
+      "əʊ": 31,
+      "uː": 32,
+      "j": 33,
+      "ɪə": 34,
+      "ɔɪ": 35,
+      "v": 36,
+      "f": 37,
+      "ɜː": 38,
+      "b": 39,
+      "pʰ": 40,
+      "d̠ʒ": 41,
+      "ɐ": 42,
+      "eə": 43,
+      "ʃ": 44,
+      "θ": 45,
+      "ŋ": 46,
+      "aʊ": 47,
+      "ʊə": 48,
+      "n̩": 49,
+      "ʒ": 50
+    },
+    "unk_token": "UNK"
+  }
+}

EnglishUK/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

EnglishUK/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd59d8993f5ebe07ee9dd53f0eec7ad0e6d3d893fe50bfc84b3915a0e3ed70e
+size 5368

EnglishUK/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"ð":4,"æ":5,"tʰ":6,"ɡ":7,"ʊ":8,"d":9,"ɑː":10,"l":11,"ɪ":12,"n":13,"eɪ":14,"t̠ʃ":15,"w":16,"ɒ":17,"ʌ":18,"z":19,"m":20,"iː":21,"aɪ":22,"h":23,"e":24,"kʰ":25,"s":26,"ə":27,"ɔː":28,"ɹ":29,"i":30,"əʊ":31,"uː":32,"j":33,"ɪə":34,"ɔɪ":35,"v":36,"f":37,"ɜː":38,"b":39,"pʰ":40,"d̠ʒ":41,"ɐ":42,"eə":43,"ʃ":44,"θ":45,"ŋ":46,"aʊ":47,"ʊə":48,"n̩":49,"ʒ":50}

Estonian/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 68
+}

Estonian/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

Estonian/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78c66010ba3c8cc625dad3ad613e7d29b20590f75e414439d1070c5ce184b5c8
+size 19295552

Estonian/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Estonian/tokenizer.json

@@ -0,0 +1,185 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "n": 4,
+      "o": 5,
+      "t": 6,
+      "ʃ": 7,
+      "a": 8,
+      "uː": 9,
+      "m": 10,
+      "u": 11,
+      "tʲ": 12,
+      "i": 13,
+      "s": 14,
+      "eː": 15,
+      "d": 16,
+      "iː": 17,
+      "k": 18,
+      "ɡ": 19,
+      "ɑ": 20,
+      "ɤ": 21,
+      "ʊ": 22,
+      "sʲ": 23,
+      "j": 24,
+      "aː": 25,
+      "h": 26,
+      "v": 27,
+      "æi": 28,
+      "kː": 29,
+      "e": 30,
+      "ɪ": 31,
+      "tː": 32,
+      "r": 33,
+      "ɛ": 34,
+      "mː": 35,
+      "p": 36,
+      "sː": 37,
+      "æ": 38,
+      "l": 39,
+      "pː": 40,
+      "yː": 41,
+      "æː": 42,
+      "b": 43,
+      "ɔ": 44,
+      "ɤː": 45,
+      "lː": 46,
+      "ø": 47,
+      "øː": 48,
+      "ŋ": 49,
+      "y": 50,
+      "oː": 51,
+      "rː": 52,
+      "ɲ": 53,
+      "nː": 54,
+      "w": 55,
+      "tʲː": 56,
+      "øɪ̯": 57,
+      "f": 58,
+      "dʲ": 59,
+      "sʲː": 60,
+      "t̠ʃ": 61,
+      "ʃː": 62,
+      "ʒ": 63,
+      "z": 64,
+      "fː": 65,
+      "dː": 66,
+      "yi": 67
+    },
+    "unk_token": "UNK"
+  }
+}

Estonian/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

Estonian/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:664a2959dae3a635b803b8e17eb3033cb9130704a8692c202c5db1488dbdcfaa
+size 5368

Estonian/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"n":4,"o":5,"t":6,"ʃ":7,"a":8,"uː":9,"m":10,"u":11,"tʲ":12,"i":13,"s":14,"eː":15,"d":16,"iː":17,"k":18,"ɡ":19,"ɑ":20,"ɤ":21,"ʊ":22,"sʲ":23,"j":24,"aː":25,"h":26,"v":27,"æi":28,"kː":29,"e":30,"ɪ":31,"tː":32,"r":33,"ɛ":34,"mː":35,"p":36,"sː":37,"æ":38,"l":39,"pː":40,"yː":41,"æː":42,"b":43,"ɔ":44,"ɤː":45,"lː":46,"ø":47,"øː":48,"ŋ":49,"y":50,"oː":51,"rː":52,"ɲ":53,"nː":54,"w":55,"tʲː":56,"øɪ̯":57,"f":58,"dʲ":59,"sʲː":60,"t̠ʃ":61,"ʃː":62,"ʒ":63,"z":64,"fː":65,"dː":66,"yi":67}

French/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 39
+}

French/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

French/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe74d8ebbd538360b2944bffb5f8ee45e043767da7d4cb815304e298f35fcb49
+size 19265856

French/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

French/tokenizer.json

@@ -0,0 +1,156 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "m": 4,
+      "a": 5,
+      "ɑ̃": 6,
+      "d": 7,
+      "ɔ": 8,
+      "n": 9,
+      "b": 10,
+      "ʁ": 11,
+      "ə": 12,
+      "ɡ": 13,
+      "ʒ": 14,
+      "i": 15,
+      "v": 16,
+      "t": 17,
+      "k": 18,
+      "o": 19,
+      "ɛ̃": 20,
+      "w": 21,
+      "y": 22,
+      "j": 23,
+      "e": 24,
+      "ɔ̃": 25,
+      "p": 26,
+      "ɛ": 27,
+      "f": 28,
+      "s": 29,
+      "z": 30,
+      "l": 31,
+      "u": 32,
+      "ʃ": 33,
+      "œ": 34,
+      "ø": 35,
+      "ɲ": 36,
+      "t̠ʃ": 37,
+      "d̠ʒ": 38
+    },
+    "unk_token": "UNK"
+  }
+}

French/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

French/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4df0c5a6fcef66bb6ccd6ef81260a418bc30e9d2b0783e39d760d5349ef33883
+size 5368

French/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"m":4,"a":5,"ɑ̃":6,"d":7,"ɔ":8,"n":9,"b":10,"ʁ":11,"ə":12,"ɡ":13,"ʒ":14,"i":15,"v":16,"t":17,"k":18,"o":19,"ɛ̃":20,"w":21,"y":22,"j":23,"e":24,"ɔ̃":25,"p":26,"ɛ":27,"f":28,"s":29,"z":30,"l":31,"u":32,"ʃ":33,"œ":34,"ø":35,"ɲ":36,"t̠ʃ":37,"d̠ʒ":38}

German/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 45
+}

German/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}

German/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e3c537e434f8ef587c16ed7d06e2c14cd2b3baa024b2a73eac0e57800f6cdb7
+size 19272000

German/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "UTT_BOUNDARY",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "PAD",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "UNK",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

German/tokenizer.json

@@ -0,0 +1,162 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "UNK",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "PAD",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "WORD_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "UTT_BOUNDARY",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": true,
+        "strip_right": true
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "UTT_BOUNDARY",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "UTT_BOUNDARY": {
+        "id": "UTT_BOUNDARY",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "UTT_BOUNDARY"
+        ]
+      }
+    }
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "UNK": 0,
+      "PAD": 1,
+      "WORD_BOUNDARY": 2,
+      "UTT_BOUNDARY": 3,
+      "aː": 4,
+      "oː": 5,
+      "a": 6,
+      "b": 7,
+      "x": 8,
+      "v": 9,
+      "øː": 10,
+      "n": 11,
+      "ɛː": 12,
+      "f": 13,
+      "l": 14,
+      "iː": 15,
+      "yː": 16,
+      "j": 17,
+      "uː": 18,
+      "h": 19,
+      "ʊ": 20,
+      "m": 21,
+      "ɔ": 22,
+      "ɪ": 23,
+      "eː": 24,
+      "ə": 25,
+      "d̺": 26,
+      "t̺ʰ": 27,
+      "ɛ": 28,
+      "ŋ": 29,
+      "ç": 30,
+      "œ": 31,
+      "kʰ": 32,
+      "ʀ": 33,
+      "ɡ": 34,
+      "pʰ": 35,
+      "ʏ": 36,
+      "s": 37,
+      "z": 38,
+      "ts": 39,
+      "ʃ": 40,
+      "ɐ": 41,
+      "pf": 42,
+      "t̠ʃ": 43,
+      "d̠ʒ": 44
+    },
+    "unk_token": "UNK"
+  }
+}

German/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "UNK",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "PAD",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "WORD_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "UTT_BOUNDARY",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "UTT_BOUNDARY",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "UTT_BOUNDARY",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "PAD",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "UNK"
+}

German/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f1a126b8a5b6a141b1a0da8c3a868ea5d4f2af38cd985968a344ea9e3688e0
+size 5368

German/vocab.json

@@ -0,0 +1 @@
+{"UNK":0,"PAD":1,"WORD_BOUNDARY":2,"UTT_BOUNDARY":3,"aː":4,"oː":5,"a":6,"b":7,"x":8,"v":9,"øː":10,"n":11,"ɛː":12,"f":13,"l":14,"iː":15,"yː":16,"j":17,"uː":18,"h":19,"ʊ":20,"m":21,"ɔ":22,"ɪ":23,"eː":24,"ə":25,"d̺":26,"t̺ʰ":27,"ɛ":28,"ŋ":29,"ç":30,"œ":31,"kʰ":32,"ʀ":33,"ɡ":34,"pʰ":35,"ʏ":36,"s":37,"z":38,"ts":39,"ʃ":40,"ɐ":41,"pf":42,"t̠ʃ":43,"d̠ʒ":44}

Indonesian/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.3,
+  "bos_token_id": 3,
+  "embd_pdrop": 0.3,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 256,
+  "n_head": 8,
+  "n_inner": 1024,
+  "n_layer": 6,
+  "n_positions": 256,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.3,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 31
+}

Indonesian/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 3,
+  "eos_token_id": 3,
+  "transformers_version": "4.44.2"
+}