gemma-2-2B-hermes-function-calling-adapter

README.md

@@ -36,7 +36,7 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0002
+- learning_rate: 0.0005
 - train_batch_size: 2
 - eval_batch_size: 8
 - seed: 3407
@@ -56,6 +56,6 @@ The following hyperparameters were used during training:
 
 - PEFT 0.12.0
 - Transformers 4.44.2
-- Pytorch 2.4.0+cu121
+- Pytorch 2.4.1+cu121
 - Datasets 3.0.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
     "down_proj",
-    "gate_proj",
+    "k_proj",
+    "up_proj",
     "q_proj",
     "v_proj",
-    "up_proj",
-    "k_proj"
+    "o_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:304c0a8bcc8dd7cfddd2d58b8f324fe19c44fe116a786f545bb6f9b678cc0521
+oid sha256:0645cbf3b72d27961ca56f1f50a32f8200bff47811d71479462b782ef78b91a5
 size 2442466784

runs/Sep23_07-00-17_7ed5bc258aab/events.out.tfevents.1727074833.7ed5bc258aab.2674.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e679c743baf5ff7b561188d46db36b9e044488a459456e8ad813fff9faeb682
+size 43799

tokenizer_config.json

@@ -2048,7 +2048,7 @@
     "<end_of_turn>"
   ],
   "bos_token": "<bos>",
-  "chat_template": "\n{# Macro used to render JSON elements as text. Used to render tool descriptions in chat #}\n{%- macro render_json(el, indent=4) -%}\n    {%- if el is string %}\n{{'\"' + el + '\"'}}\n    {%- elif el is mapping %}\n        {%- for key, value in el.items() %}\n            {%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n            {%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n            {%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n'}}\n                {%- for v in value %}\n{{- \" \" * (indent + 4) + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n                {%- endfor %} {# FOR LOOP values #}\n{{ '\n' + \" \" * indent + \"]\" }}\n            {%- else -%}\n{{ \" \" * indent + '\"' + key + '\" :' + value|string }}\n            {%- endif -%}\n{%- if not loop.last %},\n{% endif %}\n        {%- endfor %} {# FOR LOOP el.items() #}\n    {%- elif el is sequence -%}\n        {%- for v in el -%}\n{{\" \" * indent + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n        {%- endfor -%}\n    {%- else -%}\n{{ \" \" * indent + el|string }}\n    {%- endif %}\n{%- endmacro %}\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_init_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_init_message = none %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{%- if system_init_message is defined -%}\n{{- \"<start_of_turn>system\" + '\n' + system_init_message}}\n    {%- if tools is not none %}\n      {{- \"[\" -}}\n        {%- for tool in tools %}\n            {# {%- set tool = tool.function %} #}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n            {%- if not loop.last %}\n            {{- \"\n,\n\" }}\n            {%- else %}\n            {{- \"\n\n\" }}\n            {% endif %}\n        {%- endfor %}\n        {{- \"]\" -}}\n    {%- else %}\n    {{- '\n\n' }}\n    {%- endif %}\n{%- else -%}\n{{ \"<start_of_turn>user\" + '\n'}}\n{%- endif -%}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"system\" %}\n        {{- \"<start_of_turn>system\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"system\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'system'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
+  "chat_template": "\n{# Macro used to render JSON elements as text. Used to render tool descriptions in chat #}\n{%- macro render_json(el, indent=4) -%}\n    {%- if el is string %}\n{{'\"' + el + '\"'}}\n    {%- elif el is mapping %}\n        {%- for key, value in el.items() %}\n            {%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n            {%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n            {%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n'}}\n                {%- for v in value %}\n{{- \" \" * (indent + 4) + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n                {%- endfor %} {# FOR LOOP values #}\n{{ '\n' + \" \" * indent + \"]\" }}\n            {%- else -%}\n{{ \" \" * indent + '\"' + key + '\" :' + value|string }}\n            {%- endif -%}\n{%- if not loop.last %},\n{% endif %}\n        {%- endfor %} {# FOR LOOP el.items() #}\n    {%- elif el is sequence -%}\n        {%- for v in el -%}\n{{\" \" * indent + render_json(v, indent + 4) }}\n{%- if not loop.last -%},\n{% endif %}\n        {%- endfor -%}\n    {%- else -%}\n{{ \" \" * indent + el|string }}\n    {%- endif %}\n{%- endmacro %}\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_init_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_init_message = none %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{%- if system_init_message is defined -%}\n{{- \"<start_of_turn>system\" + '\n' + system_init_message}}\n    {%- if tools is not none %}\n      {{- \"<tools>[\" -}}\n        {%- for tool in tools %}\n            {# {%- set tool = tool.function %} #}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n            {%- if not loop.last %}\n            {{- \"\n,\n\" }}\n            {%- else %}\n            {{- \"\n\n\" }}\n            {% endif %}\n        {%- endfor %}\n        {{- \"]</tools>\" -}}\n    {%- else %}\n    {{- '\n\n' }}\n    {%- endif %}\n{%- else -%}\n{{ \"<start_of_turn>user\" + '\n'}}\n{%- endif -%}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"system\" %}\n        {{- \"<start_of_turn>system\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"system\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'system'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
   "model_max_length": 8192,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7c7ec68d70b521d5c68f649d81aa094c8c3d9c3c8bec54ac7e291ea81a8995e
+oid sha256:fe6497e601cf2fb129e59a2d7925e04e03dddce9487aee92d124a426fcc8f6d0
 size 5496