Upload LoRA model and probe head

.gitattributes

@@ -90,3 +90,7 @@ qwen2_5_7b_lora_lambda_kl_0_05/llama3_1_8b_trivia_qa_test_roc_curves.png filter=
 qwen2_5_7b_lora_lambda_kl_0_05/qwen_2_5_7b_longfact_augmented_test_roc_curves.png filter=lfs diff=lfs merge=lfs -text
 qwen2_5_7b_lora_lambda_kl_0_05/qwen_2_5_7b_longfact_test_roc_curves.png filter=lfs diff=lfs merge=lfs -text
 qwen2_5_7b_lora_lambda_kl_0_05/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+qwen2_5_7b_linear/llama3_1_8b_validation_roc_curves.png filter=lfs diff=lfs merge=lfs -text
+qwen2_5_7b_linear/qwen_2_5_7b_longfact_augmented_test_roc_curves.png filter=lfs diff=lfs merge=lfs -text
+qwen2_5_7b_linear/qwen_2_5_7b_longfact_test_roc_curves.png filter=lfs diff=lfs merge=lfs -text
+qwen2_5_7b_linear/tokenizer.json filter=lfs diff=lfs merge=lfs -text

qwen2_5_7b_linear/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

qwen2_5_7b_linear/eval_metrics.jsonl

@@ -0,0 +1,3 @@
+{"llama3_1_8b_validation/lm_loss": 0.5560820609331131, "llama3_1_8b_validation/probe_loss": 1.5419570376475653, "llama3_1_8b_validation/sparsity": 0.10468446289499601, "llama3_1_8b_validation/probe_threshold": 0.5, "llama3_1_8b_validation/all_tokens_accuracy": 0.9092038462171647, "llama3_1_8b_validation/all_tokens_precision": 0.25691853371293166, "llama3_1_8b_validation/all_tokens_recall": 0.32867322701051915, "llama3_1_8b_validation/all_tokens_f1": 0.28839969828099565, "llama3_1_8b_validation/all_tokens_auc": 0.7731135738105257, "llama3_1_8b_validation/entity_tokens_accuracy": 0.6509029242835657, "llama3_1_8b_validation/entity_tokens_precision": 0.7474918968976694, "llama3_1_8b_validation/entity_tokens_recall": 0.32867322701051915, "llama3_1_8b_validation/entity_tokens_f1": 0.45658527387574244, "llama3_1_8b_validation/entity_tokens_auc": 0.7401320674336077, "llama3_1_8b_validation/entity_span_max_accuracy": 0.7335371765232307, "llama3_1_8b_validation/entity_span_max_precision": 0.620702005730659, "llama3_1_8b_validation/entity_span_max_recall": 0.7043283885389149, "llama3_1_8b_validation/entity_span_max_f1": 0.6598762494050452, "llama3_1_8b_validation/entity_span_max_auc": 0.8008913237254176, "epoch": 1.0, "global_step": 813, "training_progress": 1.0, "dataset_id": "llama3_1_8b_validation"}
+{"qwen_2_5_7b_longfact_test/lm_loss": 0.3179175300944236, "qwen_2_5_7b_longfact_test/probe_loss": 1.3773158870397075, "qwen_2_5_7b_longfact_test/sparsity": 0.07444727889472438, "qwen_2_5_7b_longfact_test/probe_threshold": 0.5, "qwen_2_5_7b_longfact_test/all_tokens_accuracy": 0.9352831138034793, "qwen_2_5_7b_longfact_test/all_tokens_precision": 0.3004152682177598, "qwen_2_5_7b_longfact_test/all_tokens_recall": 0.27846766326557515, "qwen_2_5_7b_longfact_test/all_tokens_f1": 0.2890254080753075, "qwen_2_5_7b_longfact_test/all_tokens_auc": 0.796272475932198, "qwen_2_5_7b_longfact_test/entity_tokens_accuracy": 0.6722548837818086, "qwen_2_5_7b_longfact_test/entity_tokens_precision": 0.7975534244506531, "qwen_2_5_7b_longfact_test/entity_tokens_recall": 0.27846766326557515, "qwen_2_5_7b_longfact_test/entity_tokens_f1": 0.41280387712030014, "qwen_2_5_7b_longfact_test/entity_tokens_auc": 0.7622796061398132, "qwen_2_5_7b_longfact_test/entity_span_max_accuracy": 0.7749410295545996, "qwen_2_5_7b_longfact_test/entity_span_max_precision": 0.689077287066246, "qwen_2_5_7b_longfact_test/entity_span_max_recall": 0.6770631538163503, "qwen_2_5_7b_longfact_test/entity_span_max_f1": 0.6830173930037131, "qwen_2_5_7b_longfact_test/entity_span_max_auc": 0.8383283788562299, "epoch": 1.0, "global_step": 813, "training_progress": 1.0, "dataset_id": "qwen_2_5_7b_longfact_test"}
+{"qwen_2_5_7b_longfact_augmented_test/lm_loss": 0.2907450769481988, "qwen_2_5_7b_longfact_augmented_test/probe_loss": 1.47165536141858, "qwen_2_5_7b_longfact_augmented_test/sparsity": 0.09718684723664975, "qwen_2_5_7b_longfact_augmented_test/probe_threshold": 0.5, "qwen_2_5_7b_longfact_augmented_test/all_tokens_accuracy": 0.9141894546003928, "qwen_2_5_7b_longfact_augmented_test/all_tokens_precision": 0.3018488010250778, "qwen_2_5_7b_longfact_augmented_test/all_tokens_recall": 0.3341607984193728, "qwen_2_5_7b_longfact_augmented_test/all_tokens_f1": 0.3171840061551778, "qwen_2_5_7b_longfact_augmented_test/all_tokens_auc": 0.7966802917379214, "qwen_2_5_7b_longfact_augmented_test/entity_tokens_accuracy": 0.6467419378643643, "qwen_2_5_7b_longfact_augmented_test/entity_tokens_precision": 0.8441259278218582, "qwen_2_5_7b_longfact_augmented_test/entity_tokens_recall": 0.3341607984193728, "qwen_2_5_7b_longfact_augmented_test/entity_tokens_f1": 0.4787863390556382, "qwen_2_5_7b_longfact_augmented_test/entity_tokens_auc": 0.7794491725606717, "qwen_2_5_7b_longfact_augmented_test/entity_span_max_accuracy": 0.7660250049613019, "qwen_2_5_7b_longfact_augmented_test/entity_span_max_precision": 0.7051783659378597, "qwen_2_5_7b_longfact_augmented_test/entity_span_max_recall": 0.7399178942284472, "qwen_2_5_7b_longfact_augmented_test/entity_span_max_f1": 0.7221305679943436, "qwen_2_5_7b_longfact_augmented_test/entity_span_max_auc": 0.8466741280913976, "epoch": 1.0, "global_step": 813, "training_progress": 1.0, "dataset_id": "qwen_2_5_7b_longfact_augmented_test"}

qwen2_5_7b_linear/eval_metrics_qwen2_5_7b_it_linear_n=2k.json

@@ -0,0 +1,60 @@
+{
+    "llama3_1_8b_validation/lm_loss": 0.5560820609331131,
+    "llama3_1_8b_validation/probe_loss": 1.5419570376475653,
+    "llama3_1_8b_validation/sparsity": 0.10468446289499601,
+    "llama3_1_8b_validation/probe_threshold": 0.5,
+    "llama3_1_8b_validation/all_tokens_accuracy": 0.9092038462171647,
+    "llama3_1_8b_validation/all_tokens_precision": 0.25691853371293166,
+    "llama3_1_8b_validation/all_tokens_recall": 0.32867322701051915,
+    "llama3_1_8b_validation/all_tokens_f1": 0.28839969828099565,
+    "llama3_1_8b_validation/all_tokens_auc": 0.7731135738105257,
+    "llama3_1_8b_validation/entity_tokens_accuracy": 0.6509029242835657,
+    "llama3_1_8b_validation/entity_tokens_precision": 0.7474918968976694,
+    "llama3_1_8b_validation/entity_tokens_recall": 0.32867322701051915,
+    "llama3_1_8b_validation/entity_tokens_f1": 0.45658527387574244,
+    "llama3_1_8b_validation/entity_tokens_auc": 0.7401320674336077,
+    "llama3_1_8b_validation/entity_span_max_accuracy": 0.7335371765232307,
+    "llama3_1_8b_validation/entity_span_max_precision": 0.620702005730659,
+    "llama3_1_8b_validation/entity_span_max_recall": 0.7043283885389149,
+    "llama3_1_8b_validation/entity_span_max_f1": 0.6598762494050452,
+    "llama3_1_8b_validation/entity_span_max_auc": 0.8008913237254176,
+    "epoch": 1.0,
+    "qwen_2_5_7b_longfact_test/lm_loss": 0.3179175300944236,
+    "qwen_2_5_7b_longfact_test/probe_loss": 1.3773158870397075,
+    "qwen_2_5_7b_longfact_test/sparsity": 0.07444727889472438,
+    "qwen_2_5_7b_longfact_test/probe_threshold": 0.5,
+    "qwen_2_5_7b_longfact_test/all_tokens_accuracy": 0.9352831138034793,
+    "qwen_2_5_7b_longfact_test/all_tokens_precision": 0.3004152682177598,
+    "qwen_2_5_7b_longfact_test/all_tokens_recall": 0.27846766326557515,
+    "qwen_2_5_7b_longfact_test/all_tokens_f1": 0.2890254080753075,
+    "qwen_2_5_7b_longfact_test/all_tokens_auc": 0.796272475932198,
+    "qwen_2_5_7b_longfact_test/entity_tokens_accuracy": 0.6722548837818086,
+    "qwen_2_5_7b_longfact_test/entity_tokens_precision": 0.7975534244506531,
+    "qwen_2_5_7b_longfact_test/entity_tokens_recall": 0.27846766326557515,
+    "qwen_2_5_7b_longfact_test/entity_tokens_f1": 0.41280387712030014,
+    "qwen_2_5_7b_longfact_test/entity_tokens_auc": 0.7622796061398132,
+    "qwen_2_5_7b_longfact_test/entity_span_max_accuracy": 0.7749410295545996,
+    "qwen_2_5_7b_longfact_test/entity_span_max_precision": 0.689077287066246,
+    "qwen_2_5_7b_longfact_test/entity_span_max_recall": 0.6770631538163503,
+    "qwen_2_5_7b_longfact_test/entity_span_max_f1": 0.6830173930037131,
+    "qwen_2_5_7b_longfact_test/entity_span_max_auc": 0.8383283788562299,
+    "qwen_2_5_7b_longfact_augmented_test/lm_loss": 0.2907450769481988,
+    "qwen_2_5_7b_longfact_augmented_test/probe_loss": 1.47165536141858,
+    "qwen_2_5_7b_longfact_augmented_test/sparsity": 0.09718684723664975,
+    "qwen_2_5_7b_longfact_augmented_test/probe_threshold": 0.5,
+    "qwen_2_5_7b_longfact_augmented_test/all_tokens_accuracy": 0.9141894546003928,
+    "qwen_2_5_7b_longfact_augmented_test/all_tokens_precision": 0.3018488010250778,
+    "qwen_2_5_7b_longfact_augmented_test/all_tokens_recall": 0.3341607984193728,
+    "qwen_2_5_7b_longfact_augmented_test/all_tokens_f1": 0.3171840061551778,
+    "qwen_2_5_7b_longfact_augmented_test/all_tokens_auc": 0.7966802917379214,
+    "qwen_2_5_7b_longfact_augmented_test/entity_tokens_accuracy": 0.6467419378643643,
+    "qwen_2_5_7b_longfact_augmented_test/entity_tokens_precision": 0.8441259278218582,
+    "qwen_2_5_7b_longfact_augmented_test/entity_tokens_recall": 0.3341607984193728,
+    "qwen_2_5_7b_longfact_augmented_test/entity_tokens_f1": 0.4787863390556382,
+    "qwen_2_5_7b_longfact_augmented_test/entity_tokens_auc": 0.7794491725606717,
+    "qwen_2_5_7b_longfact_augmented_test/entity_span_max_accuracy": 0.7660250049613019,
+    "qwen_2_5_7b_longfact_augmented_test/entity_span_max_precision": 0.7051783659378597,
+    "qwen_2_5_7b_longfact_augmented_test/entity_span_max_recall": 0.7399178942284472,
+    "qwen_2_5_7b_longfact_augmented_test/entity_span_max_f1": 0.7221305679943436,
+    "qwen_2_5_7b_longfact_augmented_test/entity_span_max_auc": 0.8466741280913976
+}

qwen2_5_7b_linear/probe_config.json

@@ -0,0 +1,5 @@
+{
+    "target_layer_name": "Qwen2DecoderLayer",
+    "layer_idx": 26,
+    "hidden_size": 3584
+}

qwen2_5_7b_linear/probe_head.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cefe22edf6178b4031830458402092bdfcea8f611af604b24971e9b22d0b3a6
+size 8682

qwen2_5_7b_linear/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

qwen2_5_7b_linear/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6baa6d25bffdfad5428b2307a6abab93795f236c080483e701825b9eee04a178
+size 11422176

qwen2_5_7b_linear/tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

qwen2_5_7b_linear/training_config.json

@@ -0,0 +1,49 @@
+{
+    "wandb_project": "probe-training",
+    "model_name": "Qwen/Qwen2.5-7B-Instruct",
+    "upload_to_hf": true,
+    "hf_repo_id": "obalcells/hallucination-probes",
+    "save_evaluation_metrics": true,
+    "evaluation_output_dir": null,
+    "probe_id": "qwen2_5_7b_it_linear_n=2k",
+    "probe_dir": "/root/hallucination_detection/value_head_probes/qwen2_5_7b_it_linear_n=2k",
+    "layer": 26,
+    "lora_r": 16,
+    "lora_alpha": 32,
+    "lora_dropout": 0.05,
+    "lora_layers": [],
+    "probe_threshold": 0.5,
+    "load_from_hf": false,
+    "load_from_disk": false,
+    "train_data_config_path": "experiments/training/qwen-2.5-7b/train_data.yaml",
+    "eval_data_config_path": "experiments/training/qwen-2.5-7b/eval_data.yaml",
+    "cache_data_config_path": null,
+    "high_loss_threshold": null,
+    "lambda_lm": 0.01,
+    "lambda_kl": 0.0,
+    "anneal_max_aggr": true,
+    "anneal_warmup": 1.0,
+    "learning_rate": 5e-05,
+    "probe_head_lr": 0.001,
+    "lora_lr": 5e-05,
+    "sparsity_penalty_weight": null,
+    "use_focal_loss": false,
+    "focal_gamma": 2.0,
+    "evaluate_only": false,
+    "max_steps": -1,
+    "num_train_epochs": 1,
+    "per_device_train_batch_size": 2,
+    "per_device_eval_batch_size": 16,
+    "logging_steps": 10,
+    "eval_steps": null,
+    "cache_steps": null,
+    "evaluation_strategy": "no",
+    "seed": 42,
+    "save_roc_curves": true,
+    "dump_raw_eval_results": false,
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "enable_gradient_checkpointing": true,
+    "gradient_accumulation_steps": 1,
+    "max_grad_norm": 1.0
+}