Add Spider SQL LoRA adapter

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,85 @@
+---
+base_model: Qwen/Qwen3.5-4B
+library_name: peft
+pipeline_tag: text-generation
+datasets:
+- spider
+tags:
+- base_model:adapter:Qwen/Qwen3.5-4B
+- lora
+- text-to-sql
+- tool-calling
+- unsloth
+---
+
+# Qwen3.5-4B Spider text-to-SQL LoRA
+
+A LoRA adapter for [Qwen/Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B) that writes SQLite queries
+from English questions. It was trained on tool-calling traces built from the
+[Spider](https://yale-lily.github.io/spider) training set.
+
+Code to rebuild the data, retrain, and run the benchmarks:
+[Self-healing-Text2SQL-Agent](https://github.com/mehul79/Self-healing-Text2SQL-Agent) (`training/` and `backend/eval/`).
+
+## Results
+
+Spider dev, execution accuracy (%), all 1,034 questions, official Spider evaluator.
+Both models run inside the Self-healing Text2SQL harness (LangGraph: generate, validate, execute, repair), temperature 0.
+
+| Model | Easy | Medium | Hard | Extra | All |
+|---|---|---|---|---|---|
+| Qwen3.5-4B base, first attempt | 87.1 | 78.0 | 56.9 | 45.2 | 71.4 |
+| Qwen3.5-4B base, with repair | 86.7 | 80.5 | 57.5 | 50.0 | 73.2 |
+| **This adapter, first attempt** | 89.9 | 77.4 | 74.1 | 51.8 | **75.7** |
+| **This adapter, with repair** | 90.3 | 80.0 | 74.7 | 53.0 | **77.3** |
+
+Most of the gain is on hard questions (+17 points).
+
+## Training
+
+Every training example has the same three-step tool-calling shape:
+
+1. `get_schema` returns the database's `CREATE TABLE` statements.
+2. `execute_sql` runs the gold query and returns up to 10 rows.
+3. `submit_answer` submits the gold query.
+
+The traces come from Spider train. Gold queries that failed to run were dropped, leaving 6,797 traces.
+200 were held out for validation (seed 42). Loss is computed only on the assistant turns, after an empty
+`<think></think>` block, so the model is trained with thinking off.
+
+| Setting | Value |
+|---|---|
+| Method | LoRA, bf16 (Unsloth) |
+| Rank / alpha / dropout | 64 / 128 / 0 |
+| Target modules | q, k, v, o, gate, up, down projections |
+| Learning rate | 1e-4, cosine schedule, 3% warmup |
+| Batch | 4 per device x 4 accumulation = 16 |
+| Epochs | 2 (850 steps) |
+| Max sequence length | 4096 |
+| Hardware | 1 GPU, 13.7 GB peak, about 75 minutes |
+| Final losses | train 0.009, validation 0.0165 |
+
+## Usage
+
+Serve the base model with the adapter in vLLM, with thinking off:
+
+```bash
+vllm serve Qwen/Qwen3.5-4B --enable-lora \
+  --lora-modules text2sql-sft=mehul79/Text2SQL-4B-Model \
+  --enable-auto-tool-choice --tool-call-parser qwen3_coder \
+  --reasoning-parser qwen3 \
+  --default-chat-template-kwargs '{"enable_thinking": false}' \
+  --language-model-only --max-model-len 16384
+```
+
+Or merge it into the base model with PEFT (`PeftModel.from_pretrained(base, "mehul79/Text2SQL-4B-Model").merge_and_unload()`)
+and serve the merged weights.
+
+## Limitations
+
+- Trained only on Spider and SQLite. Other SQL dialects and real-world schemas are untested.
+- The validation split comes from Spider train, so the validation loss says little about new databases;
+  the Spider dev results above are the real measure.
+- Every training trace has one fixed shape and contains no errors, so the model never saw a failed query
+  being fixed during training.
+- It sometimes writes Spider-style double-quoted string literals (`"value"`), which strict SQL parsers reject.

adapter_config.json

@@ -0,0 +1,47 @@
+meh{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": {
+    "base_model_class": "Qwen3_5ForConditionalGeneration",
+    "parent_library": "transformers.models.qwen3_5.modeling_qwen3_5",
+    "unsloth_fixed": true
+  },
+  "base_model_name_or_path": "Qwen/Qwen3.5-4B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "kasa_config": null,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "lora_ga_config": null,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "monteclora_config": null,
+  "peft_type": "LORA",
+  "peft_version": "0.21.2",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(?:.*?(?:language|text).*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?\\.(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj))|(?:\\bmodel\\.layers\\.[\\d]{1,}(?!.*\\.experts\\.\\d)\\..*?(?:self_attn|attention|attn|mixer|mlp|feed_forward|ffn|dense|mixer).*?\\.(?:(?:q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)))",
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_bdlora": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false,
+  "velora_config": null
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:627be94a79260f2f7fec10738292b4e29d71a38b6fa2129db7eb931a08ff080d
+size 339776632

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523
+size 19989325

tokenizer_config.json

@@ -0,0 +1,32 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

train_log.jsonl

@@ -0,0 +1,95 @@
+{"step": 10, "epoch": 0.023529411764705882, "time": 1793036132.370058, "loss": 0.22702066898345946, "grad_norm": 0.6640090942382812, "learning_rate": 3.6e-05, "gpu_mem_gb": 13.36}
+{"step": 20, "epoch": 0.047058823529411764, "time": 1793036185.654046, "loss": 0.06938545107841491, "grad_norm": 0.47914600372314453, "learning_rate": 7.6e-05, "gpu_mem_gb": 13.5}
+{"step": 30, "epoch": 0.07058823529411765, "time": 1793036236.498706, "loss": 0.0575491726398468, "grad_norm": 0.27023765444755554, "learning_rate": 9.999419979367214e-05, "gpu_mem_gb": 13.5}
+{"step": 40, "epoch": 0.09411764705882353, "time": 1793036285.0721498, "loss": 0.05584830045700073, "grad_norm": 0.3506012260913849, "learning_rate": 9.992896292597229e-05, "gpu_mem_gb": 13.5}
+{"step": 50, "epoch": 0.11764705882352941, "time": 1793036338.9749413, "loss": 0.046492868661880495, "grad_norm": 0.30387771129608154, "learning_rate": 9.97913338352859e-05, "gpu_mem_gb": 13.5}
+{"step": 60, "epoch": 0.1411764705882353, "time": 1793036387.087952, "loss": 0.05704301595687866, "grad_norm": 0.5013608336448669, "learning_rate": 9.958151207063704e-05, "gpu_mem_gb": 13.5}
+{"step": 70, "epoch": 0.16470588235294117, "time": 1793036437.4532344, "loss": 0.04895034432411194, "grad_norm": 0.2968132197856903, "learning_rate": 9.929980185352526e-05, "gpu_mem_gb": 13.5}
+{"step": 80, "epoch": 0.18823529411764706, "time": 1793036485.3591592, "loss": 0.044801142811775205, "grad_norm": 0.2740367650985718, "learning_rate": 9.894661163683361e-05, "gpu_mem_gb": 13.5}
+{"step": 90, "epoch": 0.21176470588235294, "time": 1793036540.420378, "loss": 0.04286765456199646, "grad_norm": 0.13535021245479584, "learning_rate": 9.852245351261099e-05, "gpu_mem_gb": 13.52}
+{"step": 100, "epoch": 0.23529411764705882, "time": 1793036588.9007075, "loss": 0.047144800424575806, "grad_norm": 0.2513267695903778, "learning_rate": 9.802794246958759e-05, "gpu_mem_gb": 13.52}
+{"step": 100, "epoch": 0.23529411764705882, "time": 1793036604.6842003, "eval_loss": 0.035334281623363495, "eval_runtime": 15.7815, "eval_samples_per_second": 6.337, "eval_steps_per_second": 1.584, "gpu_mem_gb": 13.52}
+{"step": 110, "epoch": 0.25882352941176473, "time": 1793036651.1330924, "loss": 0.03909997045993805, "grad_norm": 0.2486964613199234, "learning_rate": 9.746379550150009e-05, "gpu_mem_gb": 13.54}
+{"step": 120, "epoch": 0.2823529411764706, "time": 1793036699.6009562, "loss": 0.038081982731819154, "grad_norm": 0.23092591762542725, "learning_rate": 9.68308305675192e-05, "gpu_mem_gb": 13.54}
+{"step": 130, "epoch": 0.3058823529411765, "time": 1793036749.0410173, "loss": 0.03753519952297211, "grad_norm": 0.2653861939907074, "learning_rate": 9.612996540628718e-05, "gpu_mem_gb": 13.54}
+{"step": 140, "epoch": 0.32941176470588235, "time": 1793036797.9871385, "loss": 0.03749722242355347, "grad_norm": 0.21498343348503113, "learning_rate": 9.536221620528442e-05, "gpu_mem_gb": 13.54}
+{"step": 150, "epoch": 0.35294117647058826, "time": 1793036845.2672334, "loss": 0.031845000386238095, "grad_norm": 0.21117058396339417, "learning_rate": 9.452869612745483e-05, "gpu_mem_gb": 13.54}
+{"step": 160, "epoch": 0.3764705882352941, "time": 1793036896.005399, "loss": 0.033359569311141965, "grad_norm": 0.20872750878334045, "learning_rate": 9.363061369722595e-05, "gpu_mem_gb": 13.54}
+{"step": 170, "epoch": 0.4, "time": 1793036952.2341025, "loss": 0.036821970343589784, "grad_norm": 0.24200093746185303, "learning_rate": 9.266927104826409e-05, "gpu_mem_gb": 13.54}
+{"step": 180, "epoch": 0.4235294117647059, "time": 1793037005.828352, "loss": 0.030702540278434755, "grad_norm": 0.18374107778072357, "learning_rate": 9.164606203550497e-05, "gpu_mem_gb": 13.54}
+{"step": 190, "epoch": 0.4470588235294118, "time": 1793037051.6120374, "loss": 0.027758315205574036, "grad_norm": 0.21995128691196442, "learning_rate": 9.056247021419735e-05, "gpu_mem_gb": 13.54}
+{"step": 200, "epoch": 0.47058823529411764, "time": 1793037096.3004305, "loss": 0.028007945418357848, "grad_norm": 0.1685820370912552, "learning_rate": 8.942006668888972e-05, "gpu_mem_gb": 13.54}
+{"step": 200, "epoch": 0.47058823529411764, "time": 1793037106.284887, "eval_loss": 0.031003303825855255, "eval_runtime": 9.9768, "eval_samples_per_second": 10.023, "eval_steps_per_second": 2.506, "gpu_mem_gb": 13.54}
+{"step": 210, "epoch": 0.49411764705882355, "time": 1793037157.2594066, "loss": 0.030912512540817262, "grad_norm": 0.18973490595817566, "learning_rate": 8.822050783547889e-05, "gpu_mem_gb": 13.54}
+{"step": 220, "epoch": 0.5176470588235295, "time": 1793037207.2781014, "loss": 0.03183888196945191, "grad_norm": 0.13943719863891602, "learning_rate": 8.696553289962339e-05, "gpu_mem_gb": 13.54}
+{"step": 230, "epoch": 0.5411764705882353, "time": 1793037257.1834896, "loss": 0.030890381336212157, "grad_norm": 0.20441843569278717, "learning_rate": 8.565696147500337e-05, "gpu_mem_gb": 13.54}
+{"step": 240, "epoch": 0.5647058823529412, "time": 1793037304.5701325, "loss": 0.02990294396877289, "grad_norm": 0.31319087743759155, "learning_rate": 8.42966908650839e-05, "gpu_mem_gb": 13.54}
+{"step": 250, "epoch": 0.5882352941176471, "time": 1793037355.6071248, "loss": 0.027320706844329835, "grad_norm": 0.19986073672771454, "learning_rate": 8.288669333220616e-05, "gpu_mem_gb": 13.54}
+{"step": 260, "epoch": 0.611764705882353, "time": 1793037405.4839106, "loss": 0.031134438514709473, "grad_norm": 0.2151864469051361, "learning_rate": 8.142901323799578e-05, "gpu_mem_gb": 13.54}
+{"step": 270, "epoch": 0.6352941176470588, "time": 1793037455.2459846, "loss": 0.02885761559009552, "grad_norm": 0.2212103009223938, "learning_rate": 7.992576407923373e-05, "gpu_mem_gb": 13.54}
+{"step": 280, "epoch": 0.6588235294117647, "time": 1793037505.3912323, "loss": 0.029626962542533875, "grad_norm": 0.25277453660964966, "learning_rate": 7.837912542348818e-05, "gpu_mem_gb": 13.54}
+{"step": 290, "epoch": 0.6823529411764706, "time": 1793037550.6336727, "loss": 0.02786014974117279, "grad_norm": 0.2689838409423828, "learning_rate": 7.679133974894983e-05, "gpu_mem_gb": 13.54}
+{"step": 300, "epoch": 0.7058823529411765, "time": 1793037596.1785831, "loss": 0.022744971513748168, "grad_norm": 0.15749776363372803, "learning_rate": 7.516470919305298e-05, "gpu_mem_gb": 13.54}
+{"step": 300, "epoch": 0.7058823529411765, "time": 1793037606.1419787, "eval_loss": 0.02623208984732628, "eval_runtime": 9.9614, "eval_samples_per_second": 10.039, "eval_steps_per_second": 2.51, "gpu_mem_gb": 13.54}
+{"step": 310, "epoch": 0.7294117647058823, "time": 1793037657.1449766, "loss": 0.027777934074401857, "grad_norm": 0.3017520308494568, "learning_rate": 7.350159221459621e-05, "gpu_mem_gb": 13.54}
+{"step": 320, "epoch": 0.7529411764705882, "time": 1793037715.1739187, "loss": 0.02481487989425659, "grad_norm": 0.11701423674821854, "learning_rate": 7.180440017420277e-05, "gpu_mem_gb": 13.59}
+{"step": 330, "epoch": 0.7764705882352941, "time": 1793037763.3904326, "loss": 0.027655673027038575, "grad_norm": 0.25778836011886597, "learning_rate": 7.007559383807803e-05, "gpu_mem_gb": 13.59}
+{"step": 340, "epoch": 0.8, "time": 1793037811.5723329, "loss": 0.026929175853729247, "grad_norm": 0.2212601751089096, "learning_rate": 6.831767981013389e-05, "gpu_mem_gb": 13.59}
+{"step": 350, "epoch": 0.8235294117647058, "time": 1793037860.8688433, "loss": 0.022548963129520417, "grad_norm": 0.19370990991592407, "learning_rate": 6.653320689765257e-05, "gpu_mem_gb": 13.59}
+{"step": 360, "epoch": 0.8470588235294118, "time": 1793037911.66784, "loss": 0.021739985048770904, "grad_norm": 0.23639102280139923, "learning_rate": 6.472476241575989e-05, "gpu_mem_gb": 13.59}
+{"step": 370, "epoch": 0.8705882352941177, "time": 1793037959.2544277, "loss": 0.028437113761901854, "grad_norm": 0.17508694529533386, "learning_rate": 6.289496843606559e-05, "gpu_mem_gb": 13.59}
+{"step": 380, "epoch": 0.8941176470588236, "time": 1793038005.3852978, "loss": 0.023317483067512513, "grad_norm": 0.08039126545190811, "learning_rate": 6.104647798491021e-05, "gpu_mem_gb": 13.59}
+{"step": 390, "epoch": 0.9176470588235294, "time": 1793038059.25213, "loss": 0.024451041221618654, "grad_norm": 0.11613956093788147, "learning_rate": 5.9181971196730454e-05, "gpu_mem_gb": 13.59}
+{"step": 400, "epoch": 0.9411764705882353, "time": 1793038112.4534595, "loss": 0.020193815231323242, "grad_norm": 0.119344063103199, "learning_rate": 5.730415142812059e-05, "gpu_mem_gb": 13.65}
+{"step": 400, "epoch": 0.9411764705882353, "time": 1793038122.4326813, "eval_loss": 0.022157657891511917, "eval_runtime": 9.977, "eval_samples_per_second": 10.023, "eval_steps_per_second": 2.506, "gpu_mem_gb": 13.65}
+{"step": 410, "epoch": 0.9647058823529412, "time": 1793038173.3237462, "loss": 0.021361738443374634, "grad_norm": 0.22612875699996948, "learning_rate": 5.5415741338223736e-05, "gpu_mem_gb": 13.65}
+{"step": 420, "epoch": 0.9882352941176471, "time": 1793038219.0601718, "loss": 0.027578166127204894, "grad_norm": 0.21408838033676147, "learning_rate": 5.351947894113646e-05, "gpu_mem_gb": 13.65}
+{"step": 430, "epoch": 1.011764705882353, "time": 1793038316.771978, "loss": 0.02046849876642227, "grad_norm": 0.16388343274593353, "learning_rate": 5.161811363605006e-05, "gpu_mem_gb": 13.65}
+{"step": 440, "epoch": 1.035294117647059, "time": 1793038368.8396392, "loss": 0.016283582150936126, "grad_norm": 0.1236100122332573, "learning_rate": 4.971440222088459e-05, "gpu_mem_gb": 13.65}
+{"step": 450, "epoch": 1.0588235294117647, "time": 1793038418.3282723, "loss": 0.01590244621038437, "grad_norm": 0.24229609966278076, "learning_rate": 4.781110489519541e-05, "gpu_mem_gb": 13.65}
+{"step": 460, "epoch": 1.0823529411764705, "time": 1793038468.638738, "loss": 0.013228027522563935, "grad_norm": 0.28009191155433655, "learning_rate": 4.591098125814776e-05, "gpu_mem_gb": 13.65}
+{"step": 470, "epoch": 1.1058823529411765, "time": 1793038521.491202, "loss": 0.01125897467136383, "grad_norm": 0.2535572648048401, "learning_rate": 4.401678630736172e-05, "gpu_mem_gb": 13.65}
+{"step": 480, "epoch": 1.1294117647058823, "time": 1793038571.2746713, "loss": 0.013785281777381897, "grad_norm": 0.21918034553527832, "learning_rate": 4.213126644442911e-05, "gpu_mem_gb": 13.65}
+{"step": 490, "epoch": 1.1529411764705881, "time": 1793038617.4371116, "loss": 0.013334038853645324, "grad_norm": 0.19969627261161804, "learning_rate": 4.0257155492893705e-05, "gpu_mem_gb": 13.65}
+{"step": 500, "epoch": 1.1764705882352942, "time": 1793038663.9579003, "loss": 0.009669050574302673, "grad_norm": 0.19586659967899323, "learning_rate": 3.8397170734468425e-05, "gpu_mem_gb": 13.65}
+{"step": 500, "epoch": 1.1764705882352942, "time": 1793038673.929371, "eval_loss": 0.022974804043769836, "eval_runtime": 9.9695, "eval_samples_per_second": 10.031, "eval_steps_per_second": 2.508, "gpu_mem_gb": 13.65}
+{"step": 510, "epoch": 1.2, "time": 1793038722.5599148, "loss": 0.01428457796573639, "grad_norm": 0.13368070125579834, "learning_rate": 3.655400896923672e-05, "gpu_mem_gb": 13.65}
+{"step": 520, "epoch": 1.223529411764706, "time": 1793038778.607055, "loss": 0.011456044763326645, "grad_norm": 0.1446235477924347, "learning_rate": 3.473034260555014e-05, "gpu_mem_gb": 13.65}
+{"step": 530, "epoch": 1.2470588235294118, "time": 1793038824.1834052, "loss": 0.010027643293142319, "grad_norm": 0.11068848520517349, "learning_rate": 3.292881578529179e-05, "gpu_mem_gb": 13.65}
+{"step": 540, "epoch": 1.2705882352941176, "time": 1793038871.9924257, "loss": 0.013342589139938354, "grad_norm": 0.13132290542125702, "learning_rate": 3.1152040550123395e-05, "gpu_mem_gb": 13.65}
+{"step": 550, "epoch": 1.2941176470588236, "time": 1793038922.717052, "loss": 0.012014560401439667, "grad_norm": 0.15210068225860596, "learning_rate": 2.9402593054274557e-05, "gpu_mem_gb": 13.65}
+{"step": 560, "epoch": 1.3176470588235294, "time": 1793038971.4509847, "loss": 0.01325671374797821, "grad_norm": 0.12173773348331451, "learning_rate": 2.7683009829365412e-05, "gpu_mem_gb": 13.65}
+{"step": 570, "epoch": 1.3411764705882354, "time": 1793039017.7151692, "loss": 0.011308324337005616, "grad_norm": 0.15462949872016907, "learning_rate": 2.5995784106678266e-05, "gpu_mem_gb": 13.65}
+{"step": 580, "epoch": 1.3647058823529412, "time": 1793039070.4773717, "loss": 0.01109156608581543, "grad_norm": 0.15264099836349487, "learning_rate": 2.434336220221067e-05, "gpu_mem_gb": 13.65}
+{"step": 590, "epoch": 1.388235294117647, "time": 1793039119.26245, "loss": 0.009209595620632172, "grad_norm": 0.17849166691303253, "learning_rate": 2.2728139969751006e-05, "gpu_mem_gb": 13.65}
+{"step": 600, "epoch": 1.4117647058823528, "time": 1793039169.3102212, "loss": 0.008871766924858093, "grad_norm": 0.13890035450458527, "learning_rate": 2.115245932711954e-05, "gpu_mem_gb": 13.65}
+{"step": 600, "epoch": 1.4117647058823528, "time": 1793039179.292682, "eval_loss": 0.019307710230350494, "eval_runtime": 9.9805, "eval_samples_per_second": 10.02, "eval_steps_per_second": 2.505, "gpu_mem_gb": 13.65}
+{"step": 610, "epoch": 1.4352941176470588, "time": 1793039229.7696133, "loss": 0.00913231447339058, "grad_norm": 0.09686557948589325, "learning_rate": 1.9618604860611556e-05, "gpu_mem_gb": 13.65}
+{"step": 620, "epoch": 1.4588235294117646, "time": 1793039281.7153718, "loss": 0.009358158707618714, "grad_norm": 0.1725200116634369, "learning_rate": 1.8128800512565513e-05, "gpu_mem_gb": 13.65}
+{"step": 630, "epoch": 1.4823529411764707, "time": 1793039330.4333432, "loss": 0.012348221987485886, "grad_norm": 0.1653304249048233, "learning_rate": 1.66852063568594e-05, "gpu_mem_gb": 13.65}
+{"step": 640, "epoch": 1.5058823529411764, "time": 1793039383.4798312, "loss": 0.009594768285751343, "grad_norm": 0.13480274379253387, "learning_rate": 1.5289915467010028e-05, "gpu_mem_gb": 13.65}
+{"step": 650, "epoch": 1.5294117647058822, "time": 1793039428.8849354, "loss": 0.009584491699934005, "grad_norm": 0.14541535079479218, "learning_rate": 1.3944950881416541e-05, "gpu_mem_gb": 13.65}
+{"step": 660, "epoch": 1.5529411764705883, "time": 1793039479.3838415, "loss": 0.00970582515001297, "grad_norm": 0.11197491735219955, "learning_rate": 1.2652262670148134e-05, "gpu_mem_gb": 13.65}
+{"step": 670, "epoch": 1.576470588235294, "time": 1793039529.0590131, "loss": 0.00817289724946022, "grad_norm": 0.24050477147102356, "learning_rate": 1.1413725107528955e-05, "gpu_mem_gb": 13.65}
+{"step": 680, "epoch": 1.6, "time": 1793039579.7988105, "loss": 0.009294144809246063, "grad_norm": 0.15437829494476318, "learning_rate": 1.0231133954619448e-05, "gpu_mem_gb": 13.65}
+{"step": 690, "epoch": 1.6235294117647059, "time": 1793039627.6768477, "loss": 0.0072763323783874515, "grad_norm": 0.17353779077529907, "learning_rate": 9.106203855534479e-06, "gpu_mem_gb": 13.65}
+{"step": 700, "epoch": 1.6470588235294117, "time": 1793039677.039185, "loss": 0.012826266884803771, "grad_norm": 0.21652153134346008, "learning_rate": 8.040565851373334e-06, "gpu_mem_gb": 13.65}
+{"step": 700, "epoch": 1.6470588235294117, "time": 1793039687.024363, "eval_loss": 0.017000731080770493, "eval_runtime": 9.9778, "eval_samples_per_second": 10.022, "eval_steps_per_second": 2.506, "gpu_mem_gb": 13.65}
+{"step": 710, "epoch": 1.6705882352941175, "time": 1793039736.8967767, "loss": 0.010262548923492432, "grad_norm": 0.1172318235039711, "learning_rate": 7.035765015366047e-06, "gpu_mem_gb": 13.65}
+{"step": 720, "epoch": 1.6941176470588235, "time": 1793039779.3593428, "loss": 0.008216558396816254, "grad_norm": 0.11666480451822281, "learning_rate": 6.093258212664937e-06, "gpu_mem_gb": 13.65}
+{"step": 730, "epoch": 1.7176470588235295, "time": 1793039829.9900742, "loss": 0.01054987609386444, "grad_norm": 0.1768999695777893, "learning_rate": 5.214411988029355e-06, "gpu_mem_gb": 13.65}
+{"step": 740, "epoch": 1.7411764705882353, "time": 1793039881.9238267, "loss": 0.009328901767730713, "grad_norm": 0.2757999300956726, "learning_rate": 4.400500584466505e-06, "gpu_mem_gb": 13.65}
+{"step": 750, "epoch": 1.7647058823529411, "time": 1793039930.1229742, "loss": 0.010568960011005402, "grad_norm": 0.1298745572566986, "learning_rate": 3.6527040957008485e-06, "gpu_mem_gb": 13.65}
+{"step": 760, "epoch": 1.788235294117647, "time": 1793039976.1028185, "loss": 0.007575752586126328, "grad_norm": 0.12025629729032516, "learning_rate": 2.972106755151027e-06, "gpu_mem_gb": 13.65}
+{"step": 770, "epoch": 1.811764705882353, "time": 1793040027.1038735, "loss": 0.009393654763698578, "grad_norm": 0.11973751336336136, "learning_rate": 2.359695363895109e-06, "gpu_mem_gb": 13.65}
+{"step": 780, "epoch": 1.835294117647059, "time": 1793040079.66815, "loss": 0.008702496439218521, "grad_norm": 0.23288004100322723, "learning_rate": 1.8163578599033005e-06, "gpu_mem_gb": 13.65}
+{"step": 790, "epoch": 1.8588235294117648, "time": 1793040127.9253604, "loss": 0.010824108123779297, "grad_norm": 0.24194037914276123, "learning_rate": 1.3428820306128075e-06, "gpu_mem_gb": 13.65}
+{"step": 800, "epoch": 1.8823529411764706, "time": 1793040178.091501, "loss": 0.008621842414140702, "grad_norm": 0.14583957195281982, "learning_rate": 9.3995437071136e-07, "gpu_mem_gb": 13.65}
+{"step": 800, "epoch": 1.8823529411764706, "time": 1793040188.0726993, "eval_loss": 0.01648063398897648, "eval_runtime": 9.9793, "eval_samples_per_second": 10.021, "eval_steps_per_second": 2.505, "gpu_mem_gb": 13.65}
+{"step": 810, "epoch": 1.9058823529411764, "time": 1793040240.1891897, "loss": 0.007448796182870865, "grad_norm": 0.1332673728466034, "learning_rate": 6.081590867855536e-07, "gpu_mem_gb": 13.65}
+{"step": 820, "epoch": 1.9294117647058824, "time": 1793040287.434673, "loss": 0.009504692256450653, "grad_norm": 0.07954863458871841, "learning_rate": 3.479772502771372e-07, "gpu_mem_gb": 13.65}
+{"step": 830, "epoch": 1.9529411764705882, "time": 1793040338.580648, "loss": 0.008373192697763442, "grad_norm": 0.08594575524330139, "learning_rate": 1.5978609997542304e-07, "gpu_mem_gb": 13.65}
+{"step": 840, "epoch": 1.9764705882352942, "time": 1793040391.803478, "loss": 0.008685307204723358, "grad_norm": 0.15327264368534088, "learning_rate": 4.385849505708084e-08, "gpu_mem_gb": 13.65}
+{"step": 850, "epoch": 2.0, "time": 1793040442.702235, "loss": 0.00900450199842453, "grad_norm": 0.1465459018945694, "learning_rate": 3.6251946647358757e-10, "gpu_mem_gb": 13.65}
+{"step": 850, "epoch": 2.0, "time": 1793040452.619821, "eval_loss": 0.016495272517204285, "eval_runtime": 9.9149, "eval_samples_per_second": 10.086, "eval_steps_per_second": 2.521, "gpu_mem_gb": 13.65}
+{"step": 850, "epoch": 2.0, "time": 1793040453.7610426, "train_runtime": 4523.1667, "train_samples_per_second": 3.005, "train_steps_per_second": 0.188, "total_flos": 5.246867589859799e+17, "train_loss": 0.024574783102554433, "gpu_mem_gb": 13.65}