Initial commit of GPT2 201920

config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "MoEGPTForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration.MoEGPTConfig",
+    "AutoModelForCausalLM": "modeling.MoEGPTForCausalLM",
+    "AutoTokenizer": "GPT2TokenizerFast"
+  },
+  "bias": false,
+  "dropout": 0.0,
+  "mlp_dim_exp_factor": 1.0,
+  "model_type": "moegpt",
+  "moe": false,
+  "moe_aux_loss_factor": 0.01,
+  "moe_num_experts": 1,
+  "moe_num_experts_per_tok": 2,
+  "moe_router_loss": "load_balancing_z_loss",
+  "moe_routing": "None",
+  "moe_softmax_order": "softmax_topk",
+  "moe_z_loss_factor": 1.0,
+  "n_embd": 1152,
+  "n_head": 16,
+  "n_layer": 24,
+  "sequence_length": 1024,
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.1",
+  "vocab_size": 50304
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b85f28151f89b6fd326a1d269da39b105b3281a45fad98320b4661ae998d11e
+size 1765615702

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}