Hugging Face's logo

bruhzair
/
prototype-0.4x330

Text Generation
Transformers
Safetensors
llama
mergekit
Merge
conversational
text-generation-inference
Model card Files
xet
Community
bruhzair commited on
Commit
b46c3c8
·
verified ·
1 Parent(s): ff7328b

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. README.md +47 -0
  3. chat_template.jinja +7 -0
  4. config.json +35 -0
  5. mergekit_config.yml +18 -0
  6. model-00001-of-00063.safetensors +3 -0
  7. model-00002-of-00063.safetensors +3 -0
  8. model-00003-of-00063.safetensors +3 -0
  9. model-00004-of-00063.safetensors +3 -0
  10. model-00005-of-00063.safetensors +3 -0
  11. model-00006-of-00063.safetensors +3 -0
  12. model-00007-of-00063.safetensors +3 -0
  13. model-00008-of-00063.safetensors +3 -0
  14. model-00009-of-00063.safetensors +3 -0
  15. model-00010-of-00063.safetensors +3 -0
  16. model-00011-of-00063.safetensors +3 -0
  17. model-00012-of-00063.safetensors +3 -0
  18. model-00013-of-00063.safetensors +3 -0
  19. model-00014-of-00063.safetensors +3 -0
  20. model-00015-of-00063.safetensors +3 -0
  21. model-00016-of-00063.safetensors +3 -0
  22. model-00017-of-00063.safetensors +3 -0
  23. model-00018-of-00063.safetensors +3 -0
  24. model-00019-of-00063.safetensors +3 -0
  25. model-00020-of-00063.safetensors +3 -0
  26. model-00021-of-00063.safetensors +3 -0
  27. model-00022-of-00063.safetensors +3 -0
  28. model-00023-of-00063.safetensors +3 -0
  29. model-00024-of-00063.safetensors +3 -0
  30. model-00025-of-00063.safetensors +3 -0
  31. model-00026-of-00063.safetensors +3 -0
  32. model-00027-of-00063.safetensors +3 -0
  33. model-00028-of-00063.safetensors +3 -0
  34. model-00029-of-00063.safetensors +3 -0
  35. model-00030-of-00063.safetensors +3 -0
  36. model-00031-of-00063.safetensors +3 -0
  37. model-00032-of-00063.safetensors +3 -0
  38. model-00033-of-00063.safetensors +3 -0
  39. model-00034-of-00063.safetensors +3 -0
  40. model-00035-of-00063.safetensors +3 -0
  41. model-00036-of-00063.safetensors +3 -0
  42. model-00037-of-00063.safetensors +3 -0
  43. model-00038-of-00063.safetensors +3 -0
  44. model-00039-of-00063.safetensors +3 -0
  45. model-00040-of-00063.safetensors +3 -0
  46. model-00041-of-00063.safetensors +3 -0
  47. model-00042-of-00063.safetensors +3 -0
  48. model-00043-of-00063.safetensors +3 -0
  49. model-00044-of-00063.safetensors +3 -0
  50. model-00045-of-00063.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: []
3
+ library_name: transformers
4
+ tags:
5
+ - mergekit
6
+ - merge
7
+
8
+ ---
9
+ # prototype-0.4x330
10
+
11
+ This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
12
+
13
+ ## Merge Details
14
+ ### Merge Method
15
+
16
+ This model was merged using the [Multi-SLERP](https://goddard.blog/posts/multislerp-wow-what-a-cool-idea) merge method using /workspace/cache/models--deepcogito--cogito-v2-preview-llama-70B/snapshots/1e1d12e8eaebd6084a8dcf45ecdeaa2f4b8879ce as a base.
17
+
18
+ ### Models Merged
19
+
20
+ The following models were included in the merge:
21
+ * /workspace/cache/models--Doctor-Shotgun--L3.3-70B-Magnum-Diamond/snapshots/a7dfb66b4469a4c9ca07ff28bccc73a44797e76c
22
+ * /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
23
+
24
+ ### Configuration
25
+
26
+ The following YAML configuration was used to produce this model:
27
+
28
+ ```yaml
29
+ models:
30
+ - model: /workspace/cache/models--Doctor-Shotgun--L3.3-70B-Magnum-Diamond/snapshots/a7dfb66b4469a4c9ca07ff28bccc73a44797e76c
31
+ parameters:
32
+ weight: [0.5]
33
+ - model: /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
34
+ parameters:
35
+ weight: [0.5]
36
+ base_model: /workspace/cache/models--deepcogito--cogito-v2-preview-llama-70B/snapshots/1e1d12e8eaebd6084a8dcf45ecdeaa2f4b8879ce
37
+ merge_method: multislerp
38
+ tokenizer:
39
+ source: base
40
+ chat_template: llama3
41
+ parameters:
42
+ normalize_weights: false
43
+ eps: 1e-9
44
+ pad_to_multiple_of: 8
45
+ int8_mask: true
46
+ dtype: float32
47
+ ```
chat_template.jinja ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {% set loop_messages = messages %}
2
+ {% for message in loop_messages %}
3
+ {% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}
4
+ {% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}
5
+ {{ content }}
6
+ {% endfor %}
7
+ {% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}
config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 128000,
8
+ "eos_token_id": 128001,
9
+ "head_dim": 128,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 8192,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 28672,
14
+ "max_position_embeddings": 131072,
15
+ "mlp_bias": false,
16
+ "model_type": "llama",
17
+ "num_attention_heads": 64,
18
+ "num_hidden_layers": 80,
19
+ "num_key_value_heads": 8,
20
+ "pretraining_tp": 1,
21
+ "rms_norm_eps": 1e-05,
22
+ "rope_scaling": {
23
+ "factor": 8.0,
24
+ "high_freq_factor": 4.0,
25
+ "low_freq_factor": 1.0,
26
+ "original_max_position_embeddings": 8192,
27
+ "rope_type": "llama3"
28
+ },
29
+ "rope_theta": 500000.0,
30
+ "tie_word_embeddings": false,
31
+ "torch_dtype": "float32",
32
+ "transformers_version": "4.55.2",
33
+ "use_cache": true,
34
+ "vocab_size": 128256
35
+ }
mergekit_config.yml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models:
2
+ - model: /workspace/cache/models--Doctor-Shotgun--L3.3-70B-Magnum-Diamond/snapshots/a7dfb66b4469a4c9ca07ff28bccc73a44797e76c
3
+ parameters:
4
+ weight: [0.5]
5
+ - model: /workspace/cache/models--TheDrummer--Anubis-70B-v1.1/snapshots/47ea1a3368e8d161b09acbc8c211ba4212e4b466
6
+ parameters:
7
+ weight: [0.5]
8
+ base_model: /workspace/cache/models--deepcogito--cogito-v2-preview-llama-70B/snapshots/1e1d12e8eaebd6084a8dcf45ecdeaa2f4b8879ce
9
+ merge_method: multislerp
10
+ tokenizer:
11
+ source: base
12
+ chat_template: llama3
13
+ parameters:
14
+ normalize_weights: false
15
+ eps: 1e-9
16
+ pad_to_multiple_of: 8
17
+ int8_mask: true
18
+ dtype: float32
model-00001-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e02131f4ad7ab691d46a85ff7e0ee1a547cdb0cfbcd6666d19977038df3aeb6d
3
+ size 3422618656
model-00002-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c53355423513ecf3bf2e80a556c503bb38ff39c4de6845ea99baa537f26b3ff
3
+ size 4202692736
model-00003-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62ccb4a0dacca456cc4181babffd958d7ec0162edf4e934b4a5e7316b0dde751
3
+ size 3388998352
model-00004-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51bcf50cd75df3ec7ba4b2632b94cc0a04d255bfd18fb074f3455cc23a0c0f8d
3
+ size 4202692744
model-00005-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cde18799a2ede73ff06094b385bedfb6427026c341ad3975e3c772cc9a7ab134
3
+ size 4194502464
model-00006-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:591df0013cec57208318382d435399d7177b764a954fd08397ddbda33b6f13f4
3
+ size 4865459480
model-00007-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c454bfea3e0436fcc922be4b09a2f2b3a704cc21cbeb7f0ef529f03244cdfc8
3
+ size 4596958144
model-00008-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d950ff53bbe3ed06a085be940e38bbcdbf970398cc5d32040411f98866eae94
3
+ size 4228057024
model-00009-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8025ddd6dff058f5b6eb01054d16a2a20450c4272f0bd917dc8ea33864487dd6
3
+ size 4865459480
model-00010-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e486d8711d117c337817ac0f31224fbe22a2c26b3bba6d2aa11d1f8ace62fdff
3
+ size 4596958144
model-00011-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92b754b402024cf082b0fc0cec5058a85a9e268c80198333d87904c9249f0227
3
+ size 4228057024
model-00012-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f408e43081316166202c00c64bf6578d1715ed58546949982d95c104a3d7cdb
3
+ size 4865459472
model-00013-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88d91a392633437e2f2dd38318a9afac82c33ac3989c6c9c7251de9e3eda3cd1
3
+ size 4596958144
model-00014-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3008f50eab80547681b524e6619d1e99a9d7938e09ee7e37af3c2572e32aff28
3
+ size 4160947928
model-00015-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce8611f2fec2ec33b28a7d65ac6f46c4cdd6f433743a49acfc81bc7a16460c00
3
+ size 4932568576
model-00016-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:69e9d8c0744ac39fe21d4124a5ecefec703ddab9ac34073eb6c7d5efb9c5c4b5
3
+ size 4630545568
model-00017-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:002f784d909a0d8169405ff6a293916a030a5e01e6ebba5204441b515c8f2bba
3
+ size 4429350616
model-00018-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce30a9a2a09dfa0c7abb24a3f4c46c1c2e894a6d44824964b44e644a3d6777a9
3
+ size 4597023904
model-00019-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8565d8d1f9a80928f7805d625745f4e2a8c0887ec39b41c4f17fe90e1d70f365
3
+ size 4362110000
model-00020-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad71584d150f06eba23847a8b00fe546eb33730ce25545c6e0dff63c79afd778
3
+ size 4731340744
model-00021-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93710aebb1cd87ae618b9b259f95c9d2dbd725d7df0517e63214f818e2e235e6
3
+ size 4597023904
model-00022-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:00a2b32579f6a63a2e40fcc0361716ebd8a91bc615fb2b6db373bef07fe1913e
3
+ size 4362110000
model-00023-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ec60941791cef306c8414ab200445b582679998c9813384f9634c61681f0304
3
+ size 4731340744
model-00024-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33e627be772032949fb698cf36571f2a87763c3f41c4174ee66048ab3e583065
3
+ size 4597023904
model-00025-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24ec6d520bd3b057b951a3495e3ebaa233c8eb3ece6ae9f3e63c4b406c6789dc
3
+ size 4362110000
model-00026-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7be70adf66a9084d747c83dabd0b6bbf4575ec1212263a2defe42e11b65e3ce7
3
+ size 4731340744
model-00027-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce153379080c9c7b6d6297fcb521bc8f4a4dc295add66823f1b763fedd30be6c
3
+ size 4596991024
model-00028-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:129f7c91a67bc9e1576a37542d487a8801e99465dc8fc7049f59c7ad319b47cb
3
+ size 4362175768
model-00029-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dbe6a8ec8d7f96c6e56caacf780e0d0bbaaec04fd74a11debe002867298cf8dc
3
+ size 4697753304
model-00030-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8199817648127a1c1f022cd8395569a15da7ee319e1097b3d4f4fc2e7a16e028
3
+ size 4630578456
model-00031-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7b818bacca90846c8e2bc16962d019f8826fb86eea946824588091f4e37d359
3
+ size 4664165888
model-00032-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8639047e63970613ea57bcc8cd810148b745f9e66bf16e3eeb99cf57e32450e0
3
+ size 4395730304
model-00033-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94282365cab6302844aebe7fe177015f7df9337af3a73f85a870011e2bfb52eb
3
+ size 4362110000
model-00034-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1df1ff8e54fb4b89e30f151b6ee292e91dc59e4f8e67a2e0820e5e9b3f36b06
3
+ size 4966188896
model-00035-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:917ef02cca0f0c9b5ca5167768f4d47e1ac0cad0f6af3e1db2c3b416a36f7c47
3
+ size 4362142872
model-00036-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afbc6badd5acd7d4fa04cee6d1d41e56feb09f58d6777c013fa2ed56ee667986
3
+ size 4362142888
model-00037-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dee9e3d6bcd69513b046750793f7284f28b5b2b453bef84de2a3bd3b795e0a9
3
+ size 4966188888
model-00038-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f6a9d742e80c9875b5f8eac333983f5469c0b0a8e1a216864a0601a2554e1a08
3
+ size 4362175752
model-00039-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:baed720a20e73768355e821e38b166ea2936fe596e7a4a98f872b057d2079861
3
+ size 4362110000
model-00040-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fcdeee85090706e62b062d0ab021b85406c91ef0a6ac2b95f8b5970689e53060
3
+ size 4966221784
model-00041-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d628e22929ed43f0550d2af9d044e0d1f5913eaaaeb6360a935ffd2f0898f781
3
+ size 4362109992
model-00042-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4c6ceb275738d75ca81908118a2fe6dba85732400488fb3768396e0210999bd
3
+ size 4362142888
model-00043-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3615ce0b1a334da48c5bb2d9543edd1d608d37ca0a738ee21e5a2906a25d1706
3
+ size 4966221784
model-00044-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9591d200a47aae21cb6593c083db05ff2b554c66a69475288eb7c3a5e5b3c81
3
+ size 4362109992
model-00045-of-00063.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:788ed32a30109487e0e2df4f244308212e93488029c7ce5d7f43d852fd244254
3
+ size 4630578456