Upload folder using huggingface_hub

README.md

@@ -1,7 +1,7 @@
 ---
 base_model:
-- vilm/Quyen-SE-v0.1
 - Qwen/Qwen1.5-0.5B
+- vilm/Quyen-SE-v0.1
 library_name: transformers
 tags:
 - mergekit
@@ -15,12 +15,12 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B) as a base.
+This model was merged using the [TIES](https://arxiv.org/abs/2306.01708) merge method using [vilm/Quyen-SE-v0.1](https://huggingface.co/vilm/Quyen-SE-v0.1) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [vilm/Quyen-SE-v0.1](https://huggingface.co/vilm/Quyen-SE-v0.1)
+* [Qwen/Qwen1.5-0.5B](https://huggingface.co/Qwen/Qwen1.5-0.5B)
 
 ### Configuration
 
@@ -28,14 +28,14 @@ The following YAML configuration was used to produce this model:
 
 ```yaml
 models:
-  - model: Qwen/Qwen1.5-0.5B
-    # no parameters necessary for base model
   - model: vilm/Quyen-SE-v0.1
+    # no parameters necessary for base model
+  - model: Qwen/Qwen1.5-0.5B
     parameters:
       density: 0.5
       weight: 0.5
 merge_method: ties
-base_model: Qwen/Qwen1.5-0.5B
+base_model: vilm/Quyen-SE-v0.1
 parameters:
   normalize: true
 dtype: float16

config.json

@@ -1,11 +1,10 @@
 {
-  "_name_or_path": "Qwen/Qwen1.5-0.5B",
+  "_name_or_path": "vilm/Quyen-SE-v0.1",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151643,
+  "eos_token_id": 151645,
   "hidden_act": "silu",
   "hidden_size": 1024,
   "initializer_range": 0.02,
@@ -18,11 +17,11 @@
   "num_key_value_heads": 16,
   "rms_norm_eps": 1e-06,
   "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
   "torch_dtype": "float16",
   "transformers_version": "4.38.2",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

mergekit_config.yml

@@ -1,12 +1,12 @@
 models:
-  - model: Qwen/Qwen1.5-0.5B
-    # no parameters necessary for base model
   - model: vilm/Quyen-SE-v0.1
+    # no parameters necessary for base model
+  - model: Qwen/Qwen1.5-0.5B
     parameters:
       density: 0.5
       weight: 0.5
 merge_method: ties
-base_model: Qwen/Qwen1.5-0.5B
+base_model: vilm/Quyen-SE-v0.1
 parameters:
   normalize: true
 dtype: float16

model-00001-of-00001.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:534bc7cbcb6db378644c49ee9b4346781f0b1527c32132498127fe7378893a97
+oid sha256:8ad093d3b407f5b332793839693c351df5a6655019ea85d3db221c3233a005c4
 size 1239173056

special_tokens_map.json

@@ -3,8 +3,15 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
+  "bos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -30,10 +30,10 @@
     "<|im_start|>",
     "<|im_end|>"
   ],
-  "bos_token": null,
+  "bos_token": "<|im_end|>",
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|im_end|>",
   "errors": "replace",
   "model_max_length": 32768,
   "pad_token": "<|endoftext|>",