Hugging Face's logo

RedDragonGecko
/
NoWay_2x70b

Safetensors
mixtral
Model card Files
xet
Community
RedDragonGecko commited on
Commit
9541f94
·
verified ·
1 Parent(s): 8e660a6

Upload 58 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. config.json +42 -0
  3. mergekit_moe_config.yml +6 -0
  4. model-00001-of-00052.safetensors +3 -0
  5. model-00002-of-00052.safetensors +3 -0
  6. model-00003-of-00052.safetensors +3 -0
  7. model-00004-of-00052.safetensors +3 -0
  8. model-00005-of-00052.safetensors +3 -0
  9. model-00006-of-00052.safetensors +3 -0
  10. model-00007-of-00052.safetensors +3 -0
  11. model-00008-of-00052.safetensors +3 -0
  12. model-00009-of-00052.safetensors +3 -0
  13. model-00010-of-00052.safetensors +3 -0
  14. model-00011-of-00052.safetensors +3 -0
  15. model-00012-of-00052.safetensors +3 -0
  16. model-00013-of-00052.safetensors +3 -0
  17. model-00014-of-00052.safetensors +3 -0
  18. model-00015-of-00052.safetensors +3 -0
  19. model-00016-of-00052.safetensors +3 -0
  20. model-00017-of-00052.safetensors +3 -0
  21. model-00018-of-00052.safetensors +3 -0
  22. model-00019-of-00052.safetensors +3 -0
  23. model-00020-of-00052.safetensors +3 -0
  24. model-00021-of-00052.safetensors +3 -0
  25. model-00022-of-00052.safetensors +3 -0
  26. model-00023-of-00052.safetensors +3 -0
  27. model-00024-of-00052.safetensors +3 -0
  28. model-00025-of-00052.safetensors +3 -0
  29. model-00026-of-00052.safetensors +3 -0
  30. model-00027-of-00052.safetensors +3 -0
  31. model-00028-of-00052.safetensors +3 -0
  32. model-00029-of-00052.safetensors +3 -0
  33. model-00030-of-00052.safetensors +3 -0
  34. model-00031-of-00052.safetensors +3 -0
  35. model-00032-of-00052.safetensors +3 -0
  36. model-00033-of-00052.safetensors +3 -0
  37. model-00034-of-00052.safetensors +3 -0
  38. model-00035-of-00052.safetensors +3 -0
  39. model-00036-of-00052.safetensors +3 -0
  40. model-00037-of-00052.safetensors +3 -0
  41. model-00038-of-00052.safetensors +3 -0
  42. model-00039-of-00052.safetensors +3 -0
  43. model-00040-of-00052.safetensors +3 -0
  44. model-00041-of-00052.safetensors +3 -0
  45. model-00042-of-00052.safetensors +3 -0
  46. model-00043-of-00052.safetensors +3 -0
  47. model-00044-of-00052.safetensors +3 -0
  48. model-00045-of-00052.safetensors +3 -0
  49. model-00046-of-00052.safetensors +3 -0
  50. model-00047-of-00052.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "LatitudeGames/Wayfarer-Large-70B-Llama-3.3",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": 128009,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 8192,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 28672,
15
+ "max_position_embeddings": 131072,
16
+ "mlp_bias": false,
17
+ "model_type": "mixtral",
18
+ "num_attention_heads": 64,
19
+ "num_experts_per_tok": 2,
20
+ "num_hidden_layers": 80,
21
+ "num_key_value_heads": 8,
22
+ "num_local_experts": 2,
23
+ "output_router_logits": false,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-05,
26
+ "rope_scaling": {
27
+ "factor": 8.0,
28
+ "high_freq_factor": 4.0,
29
+ "low_freq_factor": 1.0,
30
+ "original_max_position_embeddings": 8192,
31
+ "rope_type": "llama3"
32
+ },
33
+ "rope_theta": 500000.0,
34
+ "router_aux_loss_coef": 0.001,
35
+ "router_jitter_noise": 0.0,
36
+ "sliding_window": null,
37
+ "tie_word_embeddings": false,
38
+ "torch_dtype": "bfloat16",
39
+ "transformers_version": "4.49.0",
40
+ "use_cache": false,
41
+ "vocab_size": 128256
42
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ base_model: LatitudeGames/Wayfarer-Large-70B-Llama-3.3
2
+ gate_mode: random
3
+ dtype: bfloat16
4
+ experts:
5
+ - source_model: LatitudeGames/Wayfarer-Large-70B-Llama-3.3
6
+ - source_model: SicariusSicariiStuff/Negative_LLAMA_70B
model-00001-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:876b6749b30ad7736f360f278f3b0b54daff7f1e3111817dd160cbecaa8f06c8
3
+ size 4752180696
model-00002-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6b581582355897a665bf8abfbdbe059f3a4e93b2f24f85bcecdb1dab857e496
3
+ size 4831906352
model-00003-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e97cb7ff14a2bf37507879e8b0e3db15c44524b0dc9cc77326e776cc33a76a28
3
+ size 4999661672
model-00004-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c4a35936906bb8ee103d63a6b60496abcfdc232b940fb0313a01a882b2682e5
3
+ size 4831889856
model-00005-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c32baadf9a53b5bb4982b0ac75e564903aff86ec71878710141bc3be7227aade
3
+ size 4831906352
model-00006-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2867e40dfcce1f28fff9e0c0383312d068d1015b0e6b46ff75ea8bcd846f019
3
+ size 4999645176
model-00007-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76c493102d01c40cd356d5ac1c48aa96720aa0324e9e9fa0057392f44c310d02
3
+ size 4831906352
model-00008-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1923a12910fccde8d63d43103eb33a494971731fd05d0a6c6ffe7f261bebbf73
3
+ size 4999645192
model-00009-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6aa86f7db81ac9526459412bdb27e0b8ba29a4e85f335daea5c36cd5a1a71d78
3
+ size 4831906376
model-00010-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93d655548d6d7dd7c84aa2d26c074cf06d02fd18f456296a5c1743fdcb79fa01
3
+ size 4999661688
model-00011-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1e04e6abb5fd5a4a9a32ee80f0acd65c156c9eb06badbcc1e2c80baffe0ea66
3
+ size 4831889880
model-00012-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43a5e09eaee2704abffc3980c393ecfc6c6bb0bd16c965b6589062fd1b0e73be
3
+ size 4831906376
model-00013-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7759f157402f08516b5ef54fd8352ad7fefa0feab5930d58f3cbe3e3655203ea
3
+ size 4999645192
model-00014-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bb57570c4f8c901c8bab7f80174cf738d275561074fc8f44ce72657bf2e4f889
3
+ size 4831906376
model-00015-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:71c2160e095cbb8bfe799deeacda4578241fa59b17a6b00cd7aece298ef401bb
3
+ size 4999645192
model-00016-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:196b296ca9446cbb8307c4f7f84231cad628b291c0298d06bc3f569501821524
3
+ size 4831906376
model-00017-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c7be2b846ee89cdf0440df0847d005db561bddb296a7216cd9f6bd8058c6cd0
3
+ size 4999661688
model-00018-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6e42ff589378af1c3cd1b912672326dd5a15ca10095120a31ece57cfce474f3
3
+ size 4831889880
model-00019-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0bca2709e10aad114e1c9dec31c7f0ec5926af813838064f98b5b4236a5f2f3
3
+ size 4831906376
model-00020-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f78924ffbf2d07aa57e3540d9c7bef7d68c4bccbd5d37a0dadc7e4f88835686d
3
+ size 4999645192
model-00021-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c996094abb459272bad1020fa3f3b19cdb8687274f4ac67414966fc6bcd44caa
3
+ size 4831906376
model-00022-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4889c22ccbfa516a7b736b44b5c292334551613c5f1fcf981c4bcbc5724f7735
3
+ size 4999645192
model-00023-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8fdf0efc05b7111738dd6ac218fc8e65a2a0447e964c7d6ad61d0645ac398a3e
3
+ size 4831906376
model-00024-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b93f51063c2868e4007a7fd33982f6ee22cb3b5f963190f2155b22f8913954c
3
+ size 4999661688
model-00025-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08c882a3b02e01e5b462d23831c4a87943a71204ca72dd6b5911a3e645ba9e7d
3
+ size 4831889880
model-00026-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:115b1c5ed4ff40d7cfdef2606d7c8e500fbb26dc48638c8ed8340e403f7f1a3b
3
+ size 4831906376
model-00027-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5d7be5a1415659dd7a45c21a73817472c44ca844f23fcb27c20b0805e73d73d
3
+ size 4999645192
model-00028-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6665697f453c53010baf2d0a74b4f2489a73235ae95933560a43a3d2606bf84f
3
+ size 4831906376
model-00029-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d6ac2f3e866b9f92d4e115f025ff9e0cf1332078a8ddd72feb45c4e38a7dc41
3
+ size 4999645192
model-00030-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:87526ccc05b95c9bbc62656eb952e10967815a4d8e32e8ba839d4809a17fd6df
3
+ size 4831906376
model-00031-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4013dc988aa5275000e428428e6a10725433a10eeebda65c6adac144d52a230c
3
+ size 4999661688
model-00032-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4921008d311516014c268c351f9db5ebc261014f4b19928b1c1746d57fb5368b
3
+ size 4831889880
model-00033-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:123b789dbf3b71d29996b9e1b4107066d8c392be6547bbea7f325011235d7450
3
+ size 4831906376
model-00034-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1075f5548a04ad6e4da209ef1a3fc73e6eb4583bce0edb2c716965b6ae69e363
3
+ size 4999645192
model-00035-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d55d8944875821715ffb7342a67d7138b79790753ef907b5331da98e62bf095c
3
+ size 4831906376
model-00036-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42cae457e34083242cd9c153c4d561a31fd6cd31db5a1e344a41b3fe20677598
3
+ size 4999645192
model-00037-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ff9f6ade58567d2c0070e3bb3a8c16524cdab954429657f9ebe49a03dd6f7d14
3
+ size 4831906376
model-00038-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a8b3451c8a072c0ed72fb9ac31d39ea180b1d0acb3f03b55bd2984e0e8cfc98a
3
+ size 4999661688
model-00039-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f008b595e394807e8e7c90983b0c9d9d138bc4ab578103d3ffb0df2bf477a2b
3
+ size 4831889880
model-00040-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c463c3c03094834a775e504b707b12535cb14769e3761cb904dab95ec0a7a520
3
+ size 4831906376
model-00041-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5a186d79a982dc19fb3bf912c7a9a11e4d0f765ef15886762671b25fc76f7ef
3
+ size 4999645192
model-00042-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cb123ed1c9863a0c192a3230c91a4d102921304ae49108f71b1d6f156b7e715
3
+ size 4831906376
model-00043-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42fbc46bb92715f4640146b0a0075e982d250f7521a0c11ad2aad86c773e0606
3
+ size 4999645192
model-00044-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c247202cf52d15f19a52e846a5204e6137c45d7654ffb3e87120e72d7043d9fd
3
+ size 4831906376
model-00045-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b5097c2f2739248df11ca87110a8cd420ab7dd470ee25f12444d013cfb8f92a
3
+ size 4999661688
model-00046-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99ffe044f0c8d1cceb4ea245733f75d538666c15d985d6b4316a1bda1e7b9515
3
+ size 4831889880
model-00047-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ae1f808641e9e3219cd99cd6a89861b902c2c7bc55eb81ed057ba964f82af978
3
+ size 4831906376