bruhzair commited on
Commit
ab28f45
·
verified ·
1 Parent(s): a76df85

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. README.md +47 -0
  3. chat_template.jinja +7 -0
  4. config.json +39 -0
  5. mergekit_config.yml +15 -0
  6. model-00001-of-00063.safetensors +3 -0
  7. model-00002-of-00063.safetensors +3 -0
  8. model-00003-of-00063.safetensors +3 -0
  9. model-00004-of-00063.safetensors +3 -0
  10. model-00005-of-00063.safetensors +3 -0
  11. model-00006-of-00063.safetensors +3 -0
  12. model-00007-of-00063.safetensors +3 -0
  13. model-00008-of-00063.safetensors +3 -0
  14. model-00009-of-00063.safetensors +3 -0
  15. model-00010-of-00063.safetensors +3 -0
  16. model-00011-of-00063.safetensors +3 -0
  17. model-00012-of-00063.safetensors +3 -0
  18. model-00013-of-00063.safetensors +3 -0
  19. model-00014-of-00063.safetensors +3 -0
  20. model-00015-of-00063.safetensors +3 -0
  21. model-00016-of-00063.safetensors +3 -0
  22. model-00017-of-00063.safetensors +3 -0
  23. model-00018-of-00063.safetensors +3 -0
  24. model-00019-of-00063.safetensors +3 -0
  25. model-00020-of-00063.safetensors +3 -0
  26. model-00021-of-00063.safetensors +3 -0
  27. model-00022-of-00063.safetensors +3 -0
  28. model-00023-of-00063.safetensors +3 -0
  29. model-00024-of-00063.safetensors +3 -0
  30. model-00025-of-00063.safetensors +3 -0
  31. model-00026-of-00063.safetensors +3 -0
  32. model-00027-of-00063.safetensors +3 -0
  33. model-00028-of-00063.safetensors +3 -0
  34. model-00029-of-00063.safetensors +3 -0
  35. model-00030-of-00063.safetensors +3 -0
  36. model-00031-of-00063.safetensors +3 -0
  37. model-00032-of-00063.safetensors +3 -0
  38. model-00033-of-00063.safetensors +3 -0
  39. model-00034-of-00063.safetensors +3 -0
  40. model-00035-of-00063.safetensors +3 -0
  41. model-00036-of-00063.safetensors +3 -0
  42. model-00037-of-00063.safetensors +3 -0
  43. model-00038-of-00063.safetensors +3 -0
  44. model-00039-of-00063.safetensors +3 -0
  45. model-00040-of-00063.safetensors +3 -0
  46. model-00041-of-00063.safetensors +3 -0
  47. model-00042-of-00063.safetensors +3 -0
  48. model-00043-of-00063.safetensors +3 -0
  49. model-00044-of-00063.safetensors +3 -0
  50. model-00045-of-00063.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: []
3
+ library_name: transformers
4
+ tags:
5
+ - mergekit
6
+ - merge
7
+
8
+ ---
9
+ # prototype-0.4x278
10
+
11
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
12
+
13
+ ## Merge Details
14
+ ### Merge Method
15
+
16
+ This model was merged using the [SCE](https://arxiv.org/abs/2408.07990) merge method using /workspace/cache/models--deepcogito--cogito-v1-preview-llama-70B/snapshots/1d624e2293b5b35f9cfd2349f8e02c7ebf32ca83 as a base.
17
+
18
+ ### Models Merged
19
+
20
+ The following models were included in the merge:
21
+ * /workspace/cache/models--Delta-Vector--Shimamura-70B/snapshots/1106f197a3ea1424512c30a8576bd718313b57c3
22
+ * /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
23
+ * /workspace/cache/models--SicariusSicariiStuff--Negative_LLAMA_70B/snapshots/f84e61554251d9355146537e39ba081c7b5580eb
24
+ * /workspace/cache/models--TheDrummer--Fallen-Llama-3.3-70B-v1/snapshots/d46ef2629f1c3cd46789a55793c5ff0af60de3e8
25
+ * /workspace/cache/models--Sao10K--L3.1-70B-Hanami-x1/snapshots/f054d970fe9119d0237ce97029e6f5b9fce630eb
26
+
27
+ ### Configuration
28
+
29
+ The following YAML configuration was used to produce this model:
30
+
31
+ ```yaml
32
+ models:
33
+ - model: /workspace/cache/models--Delta-Vector--Shimamura-70B/snapshots/1106f197a3ea1424512c30a8576bd718313b57c3
34
+ - model: /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
35
+ - model: /workspace/cache/models--Sao10K--L3.1-70B-Hanami-x1/snapshots/f054d970fe9119d0237ce97029e6f5b9fce630eb
36
+ - model: /workspace/cache/models--SicariusSicariiStuff--Negative_LLAMA_70B/snapshots/f84e61554251d9355146537e39ba081c7b5580eb
37
+ - model: /workspace/cache/models--TheDrummer--Fallen-Llama-3.3-70B-v1/snapshots/d46ef2629f1c3cd46789a55793c5ff0af60de3e8
38
+ base_model: /workspace/cache/models--deepcogito--cogito-v1-preview-llama-70B/snapshots/1d624e2293b5b35f9cfd2349f8e02c7ebf32ca83
39
+ select_topk: 0.24
40
+ merge_method: sce
41
+ tokenizer:
42
+ source: base
43
+ chat_template: llama3
44
+ pad_to_multiple_of: 8
45
+ int8_mask: true
46
+ dtype: float32
47
+ ```
chat_template.jinja ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {% set loop_messages = messages %}
2
+ {% for message in loop_messages %}
3
+ {% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}
4
+ {% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}
5
+ {{ content }}
6
+ {% endfor %}
7
+ {% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}
config.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 128000,
8
+ "eos_token_id": [
9
+ 128001,
10
+ 128008,
11
+ 128009
12
+ ],
13
+ "head_dim": 128,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 8192,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 28672,
18
+ "max_position_embeddings": 131072,
19
+ "mlp_bias": false,
20
+ "model_type": "llama",
21
+ "num_attention_heads": 64,
22
+ "num_hidden_layers": 80,
23
+ "num_key_value_heads": 8,
24
+ "pretraining_tp": 1,
25
+ "rms_norm_eps": 1e-05,
26
+ "rope_scaling": {
27
+ "factor": 8.0,
28
+ "high_freq_factor": 4.0,
29
+ "low_freq_factor": 1.0,
30
+ "original_max_position_embeddings": 8192,
31
+ "rope_type": "llama3"
32
+ },
33
+ "rope_theta": 500000.0,
34
+ "tie_word_embeddings": false,
35
+ "torch_dtype": "float32",
36
+ "transformers_version": "4.54.0",
37
+ "use_cache": true,
38
+ "vocab_size": 128256
39
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models:
2
+ - model: /workspace/cache/models--Delta-Vector--Shimamura-70B/snapshots/1106f197a3ea1424512c30a8576bd718313b57c3
3
+ - model: /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
4
+ - model: /workspace/cache/models--Sao10K--L3.1-70B-Hanami-x1/snapshots/f054d970fe9119d0237ce97029e6f5b9fce630eb
5
+ - model: /workspace/cache/models--SicariusSicariiStuff--Negative_LLAMA_70B/snapshots/f84e61554251d9355146537e39ba081c7b5580eb
6
+ - model: /workspace/cache/models--TheDrummer--Fallen-Llama-3.3-70B-v1/snapshots/d46ef2629f1c3cd46789a55793c5ff0af60de3e8
7
+ base_model: /workspace/cache/models--deepcogito--cogito-v1-preview-llama-70B/snapshots/1d624e2293b5b35f9cfd2349f8e02c7ebf32ca83
8
+ select_topk: 0.24
9
+ merge_method: sce
10
+ tokenizer:
11
+ source: base
12
+ chat_template: llama3
13
+ pad_to_multiple_of: 8
14
+ int8_mask: true
15
+ dtype: float32
model-00001-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d5460fa455581246dee5c4fac5e995bec9b0c043490823110f4383d2f0f8cba
3
+ size 4395763176
model-00002-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b216d56d1ba159d66c63a618bb69578a0c6d0b83606a132ce40996f9064c795
3
+ size 4697720440
model-00003-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef2bcc788f6529c892405a53c45a4ba4d5cb5ffe5ee2ead596faf8208919fe9c
3
+ size 4127228976
model-00004-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c40e6d034017b8d2f6b77061804a3e6cd090f270272dd092728aa86da0b88550
3
+ size 1241547248
model-00005-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd363122436becdac5c726079b2ad1d69416ca0693e8356c6e2bac220d9d39f2
3
+ size 4471128312
model-00006-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:922809f5fbf7dee2925427d6e1b30c85ef1437413d074be8a52a60a450fa8360
3
+ size 4429284872
model-00007-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6eddaa59db69a55b08f0fbf4868d67bbe97fd13283f2aa4ddb976cddc63d3c24
3
+ size 4093674432
model-00008-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a79822b34f406f67242d560a4557c8f5198ea421df595c34dd6a30684189046
3
+ size 3154183104
model-00009-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d6d5d553371462bb5609a453e4c7a79231947b7178fa201ab0e2fd1acacd5ec
3
+ size 4202692744
model-00010-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25ab71069df437f2b95cb341d546607f28d2936ceec4d1d374631ff3e6d53f06
3
+ size 4966090248
model-00011-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef6549f7858c9c62e2350604d03cd24745adc301ad82e4924d2fc9118d2ec303
3
+ size 4630644216
model-00012-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:23aa76bf3aafd08123f630659054e7410e09a0d1ef50bee73e4b23f3e66907c8
3
+ size 4630611328
model-00013-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebd8c58a945e774cb8eb2d1aa1af1b034f69de5494e6453f1bef821c0238a8ff
3
+ size 4429284872
model-00014-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:250344c37f0a3c20ce863340138c0d9cd4b3d9da9dd801a9b18f71de1fff768f
3
+ size 4362142880
model-00015-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c08f6fc0b18a0f54ee1ac024b111a32bf7f52f86f1c55777d03cf07bf8617035
3
+ size 4899014032
model-00016-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de658b50de54bdd97f41fb73eb13b771ddeb36b554c1ad3a6a1c34c747396513
3
+ size 4697753312
model-00017-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c6f96ff9bb4b58fafce0b851eeb00663faf0e951378dbb9384466b5805b8a93a
3
+ size 4093740184
model-00018-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:811015b6033b13126673d91daf1c98592f93c50a0f315a5812a34d802d51a1a2
3
+ size 4630545576
model-00019-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5464773ae8e83f08057d70aaa7dee7f0c31ade7afc96782ff91830a942b083eb
3
+ size 4966155992
model-00020-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68040cf797c2f7e98494006c78e143a6954e7d725ea8353203e2d76ae594af41
3
+ size 4395763192
model-00021-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:720209fc8a73bfdd6c22fc76afc7f3671697ef88cda634ef65894b0bae3e7308
3
+ size 4597023904
model-00022-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dd9b10b466262eed2ac40a142b078f95a115bce5505744bb6c2e8296250b0cd0
3
+ size 4395697432
model-00023-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e13361ae8efc6ea26a156eca56d0ad89c9e8d16494d1ef1a5ac425b47025a6f
3
+ size 4630644224
model-00024-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1c3d96d2edbba8fa70393730ccff90db1d976752a8c114857e1029eed62dede
3
+ size 4664133008
model-00025-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9106a2b49900bebae2215f2e02b66fbaf48bd13307471383957e7bc8d1063fe0
3
+ size 4362142888
model-00026-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4fe07777e2ff675208929b50e23534ab9b8edd2d2af0bd3f1407ba4728307e3a
3
+ size 4630644216
model-00027-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d8d4e47fe74c9d5b5bbf68824be274c57ce43cd3a54dfe15689763b3c1247dd
3
+ size 4932601464
model-00028-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e92ed6158264aa9150b0b7939d747a7ed3ffaa177f390299b11927a2a4bcf6fc
3
+ size 4664165872
model-00029-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3249dc8e23073a98a6d1932da881d357a9c02d13553a9dbc0e53ce695de3825
3
+ size 4999676976
model-00030-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ee3645f048e02317cf1f30167a9b2f7a945172b0083a4001d6f3b289fdc0597e
3
+ size 4932601464
model-00031-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1461c50547a2b3ea26b6a4be9ee84f313a78175e0e66dbdd8e214da2e6abe02a
3
+ size 4697753320
model-00032-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e40bc1fa54f9459264096e29e8c823b485131e9f0e319a10e7e203d423b966a
3
+ size 4697654672
model-00033-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fbe9af8827b143df67410edc88ed361224fe1c83c180adf6b077f4027fc490d
3
+ size 4328621224
model-00034-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1dfbbc5cf9e451eb2c0ee9aaf298d8c708d045162dc6d2feb28de5ce36a6b022
3
+ size 4664198768
model-00035-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:596a38e82bda6660125e7fd4b39c8b332a610e17334293fe0cf78139451602c9
3
+ size 4395664544
model-00036-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24c2f2293ccb233b4167192f7b2dcaafc66f5fa0d742c20d37d501ae978a73d4
3
+ size 4630611344
model-00037-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1da552b5cde983003a8bda4b5bb235ab96fe20c2a5f536abb37bc9cf2f22ab7d
3
+ size 4664198768
model-00038-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cba761b099f8767944293644cfd815c390b7a490ea56e6c062eb93c4fd21f5c4
3
+ size 4697687552
model-00039-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4d718079fead45e607a18af1b187c2ae8da884913d56334c4634c874e285ef6
3
+ size 4999676968
model-00040-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:856055ff6aed536d12f27f93a243df95f2a9a4fa1322dc0e13227c42a5f9887c
3
+ size 4932634344
model-00041-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f105e8240a4de8e88d804a484d83b229269037e9988a292202fb76947d5b1742
3
+ size 4697720432
model-00042-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4abb1b192de0a7f3a78afbdadf9305c30f3ce91465f56141f2a212d33fa1cbbd
3
+ size 4697720440
model-00043-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17c3c4a0935e07da30394f107b0e91d611bd819150b852dbc0cc98fc8518294b
3
+ size 4295000896
model-00044-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8344b8c0bb6e03f763a4604edeefc9833fdd4f16787a1c851270db303d4b2fc0
3
+ size 4966188888
model-00045-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30da906c2bfa0c1d1e64e4410587ba6d7181541e65206deb9c7f116ff983ee95
3
+ size 4395730320