Upload folder using huggingface_hub

README.md

@@ -1,7 +1,7 @@
 ---
 base_model:
-- Qwen/Qwen2.5-Coder-7B-Instruct
-- ClaudioItaly/Intelligence-7
+- happzy2633/qwen2.5-7b-ins-v3
+- AIDC-AI/Marco-o1
 library_name: transformers
 tags:
 - mergekit
@@ -15,13 +15,12 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the SLERP merge method.
+This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [happzy2633/qwen2.5-7b-ins-v3](https://huggingface.co/happzy2633/qwen2.5-7b-ins-v3) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct)
-* [ClaudioItaly/Intelligence-7](https://huggingface.co/ClaudioItaly/Intelligence-7)
+* [AIDC-AI/Marco-o1](https://huggingface.co/AIDC-AI/Marco-o1)
 
 ### Configuration
 
@@ -29,12 +28,25 @@ The following YAML configuration was used to produce this model:
 
 ```yaml
 models:
-  - model: ClaudioItaly/Intelligence-7
-  - model: Qwen/Qwen2.5-Coder-7B-Instruct
-merge_method: slerp
-base_model: ClaudioItaly/Intelligence-7
-dtype: bfloat16
+  - model: AIDC-AI/Marco-o1
+    parameters:
+      density: [1, 0.7, 0.1] # density gradient
+      weight: 1.0
+  - model: happzy2633/qwen2.5-7b-ins-v3
+    parameters:
+      density: 0.5
+      weight: [0, 0.3, 0.7, 1] # weight gradient
+  - model: AIDC-AI/Marco-o1
+    parameters:
+      density: 0.33
+      weight:
+        - filter: mlp
+          value: 0.5
+        - value: 0
+merge_method: ties
+base_model: happzy2633/qwen2.5-7b-ins-v3
 parameters:
-  t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers
-
+  normalize: true
+  int8_mask: true
+dtype: float16
 ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "ClaudioItaly/Intelligence-7",
+  "_name_or_path": "happzy2633/qwen2.5-7b-ins-v3",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
@@ -21,9 +21,9 @@
   "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.46.2",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 152064
 }

mergekit_config.yml

@@ -1,8 +1,22 @@
 models:
-  - model: ClaudioItaly/Intelligence-7
-  - model: Qwen/Qwen2.5-Coder-7B-Instruct
-merge_method: slerp
-base_model: ClaudioItaly/Intelligence-7
-dtype: bfloat16
+  - model: AIDC-AI/Marco-o1
+    parameters:
+      density: [1, 0.7, 0.1] # density gradient
+      weight: 1.0
+  - model: happzy2633/qwen2.5-7b-ins-v3
+    parameters:
+      density: 0.5
+      weight: [0, 0.3, 0.7, 1] # weight gradient
+  - model: AIDC-AI/Marco-o1
+    parameters:
+      density: 0.33
+      weight:
+        - filter: mlp
+          value: 0.5
+        - value: 0
+merge_method: ties
+base_model: happzy2633/qwen2.5-7b-ins-v3
 parameters:
-  t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers
+  normalize: true
+  int8_mask: true
+dtype: float16

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:868f362bf5a60173703527462063b6f7523b2b075e9fece5ba579e8dd75ee526
-size 4976698776
+oid sha256:2496e72dc061417a212b6c6d83bc42006cac006efd9e38148fb7dca4fa4da3cd
+size 4976698704

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21cf4ac72358236de0eb72b61810e54de3739b20bbd97b63f8785012641138f8
-size 4932751032
+oid sha256:558f8e670dafdf536fce38d7200bb31e6292cdd054ad8433aa9300791e39cee4
+size 4932750912

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bd4825bd841e2a436484a476c749adc31ac324cf9d4c2467a779f8908ef3f4a
-size 4991495808
+oid sha256:abe7b5ace24afc233903816dfa84e0303dc82e37ae6914bb8569e8e1befd9746
+size 4991495680

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f979133974a6999218827d537e4a71c747016445f149f13054f7d4d435f95968
-size 330326240
+oid sha256:4ac9e5d2c7fc9b8a870117239346888729c0342611979a3fc021ec8cf141cf8e
+size 330326224

special_tokens_map.json

@@ -1,7 +1,18 @@
 {
   "additional_special_tokens": [
     "<|im_start|>",
-    "<|im_end|>"
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
   ],
   "eos_token": {
     "content": "<|im_end|>",

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83396048d512ec1f3178af0d7c1f79a226bba041822614b0e26a4fd2d4b55bf7
-size 11421995
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_bos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "151643": {
@@ -180,20 +181,28 @@
   },
   "additional_special_tokens": [
     "<|im_start|>",
-    "<|im_end|>"
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\n\n你是一个经过良好训练的AI助手，你的名字是Marco-o1.由阿里国际数字商业集团的AI Business创造.\n        \n## 重要！！！！！\n当你回答问题时，你的思考应该在<Thought>内完成，<Output>内输出你的结果。\n<Thought>应该尽可能是英文，但是有2个特例，一个是对原文中的引用，另一个是是数学应该使用markdown格式，<Output>内的输出需要遵循用户输入的语言。\n        <|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "max_length": 4096,
   "model_max_length": 131072,
   "pad_token": "<|endoftext|>",
+  "padding_side": "right",
   "split_special_tokens": false,
-  "stride": 0,
   "tokenizer_class": "Qwen2Tokenizer",
-  "truncation_side": "right",
-  "truncation_strategy": "longest_first",
   "unk_token": null
 }