Upload folder using huggingface_hub

config.json

@@ -0,0 +1,34 @@
+{
+  "_name_or_path": "cognitivecomputations/WestLake-7B-v2-laser",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 4,
+  "output_router_logits": false,
+  "pad_token_id": 2,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "unsloth_version": "2024.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_moe_config.yml

@@ -0,0 +1,43 @@
+base_model: cognitivecomputations/WestLake-7B-v2-laser
+gate_mode: hidden
+dtype: bfloat16
+experts:
+  - source_model: mlabonne/Daredevil-7B
+    positive_prompts:
+      - "Answer this question from the ARC (Argument Reasoning Comprehension)."
+      - "Use common sense and logical reasoning skills."
+    negative_prompts:
+      - "nonsense"
+      - "irrational"
+      - "math"
+      - "code"
+  - source_model: cognitivecomputations/WestLake-7B-v2-laser
+    positive_prompts:
+      - "Answer this question from the Winogrande test."
+      - "Use advanced knowledge of culture and humanity"
+    negative_prompts:
+      - "ignorance"
+      - "uninformed"
+      - "creativity"
+  - source_model: mlabonne/Daredevil-7B
+    positive_prompts:
+      - "Calculate the answer to this math problem"
+      - "My mathematical capabilities are strong, allowing me to handle complex mathematical queries"
+      - "solve for"
+    negative_prompts:
+      - "incorrect"
+      - "inaccurate"
+      - "creativity"
+  - source_model: BioMistral/BioMistral-7B
+    positive_prompts:
+      - "Provide a detailed explanation based on medical literature."
+      - "Apply clinical reasoning to answer this healthcare question."
+      - "Use biomedical knowledge to solve this problem."
+      - "Interpret these clinical findings in the context of patient care."
+      - "Leverage your understanding of healthcare systems to advise on policy."
+    negative_prompts:
+      - "vague"
+      - "non-scientific"
+      - "unrelated to healthcare"
+      - "simplistic understanding"
+      - "ignore clinical guidelines"

model-00001-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37e5c11f90b5e166f4dc663e8db606f8403e17cfb66a4b332187750e00eaffaa
+size 933250496

model-00002-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7fa60d0977a800aebb57c12fba5798fc307b3ac29c478cb98de31e1de25d98
+size 905987672

model-00003-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1156308948a15fb787156a80e52e870a62bd9d09c3680ff8a4da42654df69683
+size 939525160

model-00004-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47848f7b8158f2f48eeabcbb9fe852eb15cad2e80344ba1c2a33b9a8e3a8b17f
+size 905987672

model-00005-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fdf28cf23bc33c21e2758198906f7cc87ab38fcc3b97fe836df8f8fd80f8adf
+size 939525160

model-00006-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:183ce32f36265cc4a30bd1677c440519cc44f6518713782c1fba00e606caba70
+size 905987672

model-00007-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:868c5411d56aaf67751b3550c660be26c82d07a9b9692fe4745e8cdc33d91d57
+size 905987672

model-00008-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40091ab77d9d1dbb397f2aa9a447d014e04a21505ae5967f4779a339f7fb5358
+size 939525160

model-00009-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39c58c0f0f6798762f963ded9e6ff11f912f5f432cce07cb8bc1648f2fb8f18a
+size 905987672

model-00010-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4703e81a82f41a7498e6b07387448922b74cf09f680cf85c3691da1e62be83e5
+size 989865456

model-00011-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07d4466e6db514f89dfe7165797770891662987f08ef0cfdc8c4aed565d52c2b
+size 973088024

model-00012-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83e16c44352c0fc69a60db7a65ed23fa0751870ee260c774de213c105becb5a7
+size 905987672

model-00013-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90d73b3fe4a73f1bcd61c76709c65f368d052b9408a8bd2821f9d6073127a964
+size 939525160

model-00014-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9540857d7a64485cc8bb4a0422f78ca6189ebdb54e5f3daff7bd962ccbcaa948
+size 905987672

model-00015-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73706adde63b49223dedbfeecdd198b48f8c5a0f581bd8ee998dc51f36e039b
+size 905987672

model-00016-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:931eaf93950a70a5abb7eec71abb1be68c605ef2918e63b0dd48c7073789de6a
+size 939525160

model-00017-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a833cad89a921b5f5ae7ad2c5ee26fe6ffb30393bd8226c452f7f1cec640d41d
+size 905987680

model-00018-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe6c9d3330ff9842dacd8cb1c0cbc6819e0ec1926052f7b77147fbd33034ed01
+size 989865472

model-00019-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5f0e60734ec19138f26ac1c0af212e29c3125a2ce52657893b7b3fed8b1cc5e
+size 973088032

model-00020-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b650d99cf73cc40a4ad1c569c8718b9fb7d370a7176b7a6ae18accc2b2211d74
+size 905987688

model-00021-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6d0ee22b5fe129f8d2ffc6c794c618c13b9da84518572a9aa8cd659908a5c64
+size 939525168

model-00022-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2953dedb57e2cb547b5276279a0a14707f1ccd09a424fe7e8d264383979bc1d
+size 905987688

model-00023-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:652cc20edace6b57ddf27ba10f23ba6189d77c9ceb6ad025023b80ec9fe89669
+size 905987688

model-00024-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6188989457032063c826970448da383a28e723560733d9d58fed4ab96964e7d8
+size 939525168

model-00025-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e744fe207e13731b647c94dbf320d9385dc17107cf3555e35db9a1b3618a5434
+size 905987688

model-00026-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b577cec1200bddb119a329b110af855deb97d71970a3b8aab672de07747c211
+size 989865472

model-00027-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05779fc999233f9e3dd47657e380ef1ef937be0d19ab4ae548bd2b48eb04088
+size 973088032

model-00028-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:836abd52d0a72c7cfb9932d75356ca957e017db1f915bd2d711a715834551403
+size 905987688

model-00029-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:246ce35a1f12470de6afb03615365120e3fe95c6f6d3981e76b801ee4148e5c4
+size 939525168

model-00030-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3528f2d2ac082db336652f5a2a84b595047e33d6f8e25a28cee3edfe0573f557
+size 905987688

model-00031-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31087950b6ded1a5280371bb2f9d710af6d5548110c17527ad9ad8346d2458ff
+size 905987688

model-00032-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47ef478c0767d51ddef0ea687e5d7130f2b8b9bd6612affc0daea3c0c317594d
+size 939525168

model-00033-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c490052856c87a6ab99d5b786264e9d526f21adab8054246058a2ec45e7a5c4d
+size 905987688

model-00034-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6face2dc872867f55eb0d189056d76e36269ed6dbde5afa9e1e4bd8122fddc8a
+size 989865472

model-00035-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17b61346a9df46b2061ad9f0548aebd254ac4363e6fcf30c2aad590a2861f72c
+size 973088032

model-00036-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fd103dccfe6e4398d00fb47832519c486cdf2a9421fe8f7f86098040a0c22aa
+size 905987688

model-00037-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f090b8c7cceebea97237d85ff5b97a8bcb33fa57182f0e3f56e0e14ace3c00b4
+size 939525168

model-00038-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:777c3e9438cc4151aa781a3f34f78a0d1d962cd60e50b8d42bc52fa7aaba5345
+size 905987688

model-00039-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a32218d676a787322efdddaa35572aa26ff532afee8546f0ad78487808e430e
+size 905987688

model-00040-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f6711376100a6282717b9f5db165690d84332c2d31a6bf90d239b181fbdd5dd
+size 939525168

model-00041-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80000daadcb35e6ed17930054276ff969d8a5f2b1ad12a595653589c1547019a
+size 905987688

model-00042-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f7e7e63ea486894605bba90dc618d73b4062f44fee275230f8fbd849db0174d
+size 989865472

model-00043-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0388103466d669e8a02644e51c111784a8d0235b7b146884292ae79653b564c3
+size 973088032

model-00044-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:852b26fc833ce6f83429da55194e4ad8ff5d44f1766aaa30864097acb6d37711
+size 905987688

model-00045-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79b0991fc85111532b9b84241f196222476de4f0c168ff5cd85e556f2d84e111
+size 939525168

model-00046-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3547cb51efb8cb2251949827e706092457f812f1d4492ac212f59b76e9490579
+size 905987688

model-00047-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55195c0cc968354f2358dd7789d07564547f9fe136874e173073317197bc17cf
+size 905987688

model-00048-of-00052.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:842368e1b3790b640c78fa79cd37b740eac90730038e3c1dd43f35a7a6c8166b
+size 939525168