First model version

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "fairseq-moe-15b",
+  "activation_function": "gelu",
+  "architectures": [
+    "FSGPTMoEForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "attention_heads": 12,
+  "batch_prioritized_routing": false,
+  "bos_token_id": 50257,
+  "classifier_dropout": 0.1,
+  "d_model": 768,
+  "embed_dropout": 0.0,
+  "eos_token_id": 50259,
+  "expert_capacity": null,
+  "ffn_dim": 3072,
+  "hidden_size": 768,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "moe_eval_capacity_token_fraction": -1.0,
+  "moe_token_dropout": 0.2,
+  "no_scale_embedding": false,
+  "normalize_router_prob_before_dropping": false,
+  "num_experts": 512,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_router_logits": true,
+  "pad_token_id": 50258,
+  "resid_dropout": 0.1,
+  "router_bias": false,
+  "router_dtype": "float32",
+  "router_ignore_padding_tokens": true,
+  "second_expert_policy": "all",
+  "sparse_step": 2,
+  "tokenizer_class": "GPT2Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "unk_token_id": 3,
+  "use_cache": true,
+  "vocab_size": 51200
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50257,
+  "eos_token_id": 50259,
+  "pad_token_id": 50258,
+  "transformers_version": "4.30.2"
+}

pytorch_model-00001-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2be460f07d87505395c16ef8c6d57f92fda072adcf5784e806ff16704edbcd7d
+size 9993386279

pytorch_model-00002-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e53b28ab5d7fdd32ce48abd534e86aec171f80358dafd4dbb9366ed82dd3da4
+size 9995062183

pytorch_model-00003-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d701d892819c60ad29a4dfa7720e9e7b3648425630982aa595bef88d62d6b811
+size 9995062247

pytorch_model-00004-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d290dd56beab73a05c190661135d27b5d28adb620b78fa64689ccacb014361
+size 9995062247

pytorch_model-00005-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86a3f7f6f67bbd020b4f515faef63ad23ff3d456295999234a6ce48773e6a5f8
+size 9995062503

pytorch_model-00006-of-00006.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a0f896baea9fe91568161f9032eaa38cc79b16ecd8f241c39349627605eecc
+size 8453823779