Upload folder using huggingface_hub

README.md

@@ -0,0 +1,49 @@
+---
+base_model:
+- tinycompany/shawty-CoT-Hindi-English
+- tinycompany/Shawty-1.4B-SFT-Stage-1
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# MergedShawty-v0.4
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the [SLERP](https://en.wikipedia.org/wiki/Slerp) merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [tinycompany/shawty-CoT-Hindi-English](https://huggingface.co/tinycompany/shawty-CoT-Hindi-English)
+* [tinycompany/Shawty-1.4B-SFT-Stage-1](https://huggingface.co/tinycompany/Shawty-1.4B-SFT-Stage-1)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+base_model: tinycompany/Shawty-1.4B-SFT-Stage-1
+merge_method: slerp
+tokenizer_source: base
+dtype: bfloat16
+parameters:
+  t:
+    - filter: self_attn
+      value: [0.2, 0.6, 0.4, 0.8, 1.0]
+    - filter: mlp
+      value: [1.0, 0.6, 0.8, 0.4, 0.2]
+    - value: 0.3 
+slices:
+  - sources:
+      - model: tinycompany/Shawty-1.4B-SFT-Stage-1
+        layer_range: [0, 28]
+      - model: tinycompany/shawty-CoT-Hindi-English
+        layer_range: [0, 28]
+
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "tinycompany/Shawty-1.4B-SFT-Stage-1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 81920
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+base_model: tinycompany/Shawty-1.4B-SFT-Stage-1
+merge_method: slerp
+tokenizer_source: base
+dtype: bfloat16
+parameters:
+  t:
+    - filter: self_attn
+      value: [0.2, 0.6, 0.4, 0.8, 1.0]
+    - filter: mlp
+      value: [1.0, 0.6, 0.8, 0.4, 0.2]
+    - value: 0.3 
+slices:
+  - sources:
+      - model: tinycompany/Shawty-1.4B-SFT-Stage-1
+        layer_range: [0, 28]
+      - model: tinycompany/shawty-CoT-Hindi-English
+        layer_range: [0, 28]

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f21c6ddf840b3a62e7aead5acf2c6126d231ef8c9ddb31403ce47f6e82b2b669
+size 2872377928

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,106 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4709": {
+      "content": "user",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "27673": {
+      "content": "system",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81912": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81913": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81914": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81915": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81916": {
+      "content": "assistant",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81917": {
+      "content": "BiBo",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81918": {
+      "content": "aloobun",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "81919": {
+      "content": "LowIQGenAI",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "chat_template": "\n{% set system_message = 'You are BiBo, a helpful and friendly AI assistant developed by aloobun and LowIQGenAI.' %}\n{%- if messages and messages[0]['role'] == 'system' %}\n    {{- '<|im_start|>system\\n' + (messages[0]['content'] if 'content' in messages[0] else messages[0]['value'] if 'value' in messages[0] else '') + '<|im_end|>\\n' }}\n{%- else %}\n    {{- '<|im_start|>system\\n' + system_message + '<|im_end|>\\n' }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = message['content'] if 'content' in message else message['value'] if 'value' in message else '' %}\n    {%- if message['role'] in ['user', 'human'] %}\n        {{- '<|im_start|>user\\n' + content + '<|im_end|>\\n' }}\n    {%- elif message['role'] in ['assistant', 'gpt'] %}\n        {{- '<|im_start|>assistant\\n' + content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "max_length": 2048,
+  "model_max_length": 8192,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "stride": 0,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first"
+}