Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training_progression.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -2,40 +2,79 @@
 license: apache-2.0
 base_model: LiquidAI/LFM2.5-1.2B-Instruct
 tags:
-- text-to-sql
-- sql
-- mlx
-- 4-bit
+  - text2sql
+  - sql
+  - fine-tuned
+  - lora
+  - mlx
+datasets:
+  - synthetic
 language:
-- en
+  - en
 pipeline_tag: text-generation
-library_name: mlx
 ---
 
-# LFM2.5-1.2B-Text2SQL (MLX 4-bit)
+# LFM2.5-1.2B-Text2SQL (MLX)
 
-MLX-optimized 4-bit quantized version for Apple Silicon.
+A fine-tuned version of [LiquidAI/LFM2.5-1.2B-Instruct](https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct) for Text-to-SQL generation.
+
+## Model Description
+
+This model was fine-tuned on 2000 synthetic Text-to-SQL examples generated using a teacher model (DeepSeek V3).
+The fine-tuning was performed using LoRA adapters with MLX on Apple Silicon, then fused into the base model.
+
+### Training Details
+
+- **Base Model**: LiquidAI/LFM2.5-1.2B-Instruct
+- **Training Data**: 2000 synthetic examples
+- **Training Method**: LoRA fine-tuning (FP16)
+- **Iterations**: 5400
+- **Hardware**: Apple Silicon (MLX)
+
+## Performance
+
+### Model Comparison
+
+![Model Comparison](model_comparison.png)
+
+| Metric | Teacher (DeepSeek V3) | Base Model | Fine-tuned |
+|--------|----------------------|------------|------------|
+| Exact Match | 60% | 48% | **72%** |
+| LLM-as-Judge | 90% | 75% | 87% |
+| ROUGE-L | 92% | 83% | **94%** |
+| BLEU | 85% | 70% | **89%** |
+| Semantic Similarity | 96% | 93% | **97%** |
+
+### Training Progression
+
+![Training Progression](training_progression.png)
+
+The model shows consistent improvement across all checkpoints with no signs of overfitting.
 
 ## Usage
 
+### MLX (Apple Silicon)
+
 ```python
 from mlx_lm import load, generate
 
-model, tokenizer = load("hybridaione/LFM2.5-1.2B-Text2SQL-MLX")
+model, tokenizer = load("furukama/LFM2.5-1.2B-Text2SQL-MLX")
 
-prompt = '''<|im_start|>system
-You are an expert SQL writer.<|im_end|>
-<|im_start|>user
-Schema:
-CREATE TABLE users (id INTEGER, name TEXT);
+# Example query
+prompt = '''CREATE TABLE employees (id INT, name VARCHAR, salary DECIMAL);
 
-Question: Count all users<|im_end|>
-<|im_start|>assistant
-'''
+Question: What are the names of employees earning more than 50000?'''
 
 response = generate(model, tokenizer, prompt=prompt, max_tokens=256)
 print(response)
 ```
 
-## Other Formats
-- **PyTorch/vLLM**: [hybridaione/LFM2.5-1.2B-Text2SQL](https://huggingface.co/hybridaione/LFM2.5-1.2B-Text2SQL)
+## Limitations
+
+- Trained on synthetic data for a specific database schema
+- Best suited for similar SQL query patterns seen during training
+- May not generalize well to very different database schemas
+
+## License
+
+This model is released under the Apache 2.0 license, following the base model's license.

config.json

@@ -48,16 +48,6 @@
     "num_hidden_layers": 16,
     "num_key_value_heads": 8,
     "pad_token_id": 0,
-    "quantization": {
-        "group_size": 64,
-        "bits": 4,
-        "mode": "affine"
-    },
-    "quantization_config": {
-        "group_size": 64,
-        "bits": 4,
-        "mode": "affine"
-    },
     "rope_theta": 1000000.0,
     "tie_embedding": true,
     "transformers_version": "4.57.2",

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa003939b5a7009c2095b03333ebdd6e5b1b12e515974b1d23df4fa08ab1957a
-size 658540250
+oid sha256:420460c9cac5ef5c07727fa98cc497c7ac7a6aa05991b103474458b91324a508
+size 2340697867

model.safetensors.index.json

@@ -1,340 +1,154 @@
 {
     "metadata": {
-        "total_size": 658503168,
+        "total_size": 2340681216,
         "total_parameters": 1170340608
     },
     "weight_map": {
-        "model.embed_tokens.biases": "model.safetensors",
-        "model.embed_tokens.scales": "model.safetensors",
         "model.embed_tokens.weight": "model.safetensors",
         "model.embedding_norm.weight": "model.safetensors",
         "model.layers.0.conv.conv.weight": "model.safetensors",
-        "model.layers.0.conv.in_proj.biases": "model.safetensors",
-        "model.layers.0.conv.in_proj.scales": "model.safetensors",
         "model.layers.0.conv.in_proj.weight": "model.safetensors",
-        "model.layers.0.conv.out_proj.biases": "model.safetensors",
-        "model.layers.0.conv.out_proj.scales": "model.safetensors",
         "model.layers.0.conv.out_proj.weight": "model.safetensors",
-        "model.layers.0.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.0.feed_forward.w1.scales": "model.safetensors",
         "model.layers.0.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.0.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.0.feed_forward.w2.scales": "model.safetensors",
         "model.layers.0.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.0.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.0.feed_forward.w3.scales": "model.safetensors",
         "model.layers.0.feed_forward.w3.weight": "model.safetensors",
         "model.layers.0.ffn_norm.weight": "model.safetensors",
         "model.layers.0.operator_norm.weight": "model.safetensors",
         "model.layers.1.conv.conv.weight": "model.safetensors",
-        "model.layers.1.conv.in_proj.biases": "model.safetensors",
-        "model.layers.1.conv.in_proj.scales": "model.safetensors",
         "model.layers.1.conv.in_proj.weight": "model.safetensors",
-        "model.layers.1.conv.out_proj.biases": "model.safetensors",
-        "model.layers.1.conv.out_proj.scales": "model.safetensors",
         "model.layers.1.conv.out_proj.weight": "model.safetensors",
-        "model.layers.1.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.1.feed_forward.w1.scales": "model.safetensors",
         "model.layers.1.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.1.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.1.feed_forward.w2.scales": "model.safetensors",
         "model.layers.1.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.1.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.1.feed_forward.w3.scales": "model.safetensors",
         "model.layers.1.feed_forward.w3.weight": "model.safetensors",
         "model.layers.1.ffn_norm.weight": "model.safetensors",
         "model.layers.1.operator_norm.weight": "model.safetensors",
-        "model.layers.10.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.10.feed_forward.w1.scales": "model.safetensors",
         "model.layers.10.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.10.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.10.feed_forward.w2.scales": "model.safetensors",
         "model.layers.10.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.10.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.10.feed_forward.w3.scales": "model.safetensors",
         "model.layers.10.feed_forward.w3.weight": "model.safetensors",
         "model.layers.10.ffn_norm.weight": "model.safetensors",
         "model.layers.10.operator_norm.weight": "model.safetensors",
         "model.layers.10.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.10.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.10.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.10.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.10.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.10.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.10.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.10.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.10.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.10.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.10.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.10.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.10.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.10.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.11.conv.conv.weight": "model.safetensors",
-        "model.layers.11.conv.in_proj.biases": "model.safetensors",
-        "model.layers.11.conv.in_proj.scales": "model.safetensors",
         "model.layers.11.conv.in_proj.weight": "model.safetensors",
-        "model.layers.11.conv.out_proj.biases": "model.safetensors",
-        "model.layers.11.conv.out_proj.scales": "model.safetensors",
         "model.layers.11.conv.out_proj.weight": "model.safetensors",
-        "model.layers.11.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.11.feed_forward.w1.scales": "model.safetensors",
         "model.layers.11.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.11.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.11.feed_forward.w2.scales": "model.safetensors",
         "model.layers.11.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.11.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.11.feed_forward.w3.scales": "model.safetensors",
         "model.layers.11.feed_forward.w3.weight": "model.safetensors",
         "model.layers.11.ffn_norm.weight": "model.safetensors",
         "model.layers.11.operator_norm.weight": "model.safetensors",
-        "model.layers.12.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.12.feed_forward.w1.scales": "model.safetensors",
         "model.layers.12.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.12.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.12.feed_forward.w2.scales": "model.safetensors",
         "model.layers.12.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.12.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.12.feed_forward.w3.scales": "model.safetensors",
         "model.layers.12.feed_forward.w3.weight": "model.safetensors",
         "model.layers.12.ffn_norm.weight": "model.safetensors",
         "model.layers.12.operator_norm.weight": "model.safetensors",
         "model.layers.12.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.12.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.12.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.12.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.12.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.12.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.12.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.12.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.12.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.12.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.12.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.12.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.12.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.12.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.13.conv.conv.weight": "model.safetensors",
-        "model.layers.13.conv.in_proj.biases": "model.safetensors",
-        "model.layers.13.conv.in_proj.scales": "model.safetensors",
         "model.layers.13.conv.in_proj.weight": "model.safetensors",
-        "model.layers.13.conv.out_proj.biases": "model.safetensors",
-        "model.layers.13.conv.out_proj.scales": "model.safetensors",
         "model.layers.13.conv.out_proj.weight": "model.safetensors",
-        "model.layers.13.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.13.feed_forward.w1.scales": "model.safetensors",
         "model.layers.13.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.13.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.13.feed_forward.w2.scales": "model.safetensors",
         "model.layers.13.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.13.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.13.feed_forward.w3.scales": "model.safetensors",
         "model.layers.13.feed_forward.w3.weight": "model.safetensors",
         "model.layers.13.ffn_norm.weight": "model.safetensors",
         "model.layers.13.operator_norm.weight": "model.safetensors",
-        "model.layers.14.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.14.feed_forward.w1.scales": "model.safetensors",
         "model.layers.14.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.14.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.14.feed_forward.w2.scales": "model.safetensors",
         "model.layers.14.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.14.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.14.feed_forward.w3.scales": "model.safetensors",
         "model.layers.14.feed_forward.w3.weight": "model.safetensors",
         "model.layers.14.ffn_norm.weight": "model.safetensors",
         "model.layers.14.operator_norm.weight": "model.safetensors",
         "model.layers.14.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.14.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.14.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.14.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.14.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.14.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.14.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.14.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.14.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.14.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.14.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.14.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.14.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.14.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.15.conv.conv.weight": "model.safetensors",
-        "model.layers.15.conv.in_proj.biases": "model.safetensors",
-        "model.layers.15.conv.in_proj.scales": "model.safetensors",
         "model.layers.15.conv.in_proj.weight": "model.safetensors",
-        "model.layers.15.conv.out_proj.biases": "model.safetensors",
-        "model.layers.15.conv.out_proj.scales": "model.safetensors",
         "model.layers.15.conv.out_proj.weight": "model.safetensors",
-        "model.layers.15.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.15.feed_forward.w1.scales": "model.safetensors",
         "model.layers.15.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.15.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.15.feed_forward.w2.scales": "model.safetensors",
         "model.layers.15.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.15.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.15.feed_forward.w3.scales": "model.safetensors",
         "model.layers.15.feed_forward.w3.weight": "model.safetensors",
         "model.layers.15.ffn_norm.weight": "model.safetensors",
         "model.layers.15.operator_norm.weight": "model.safetensors",
-        "model.layers.2.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.2.feed_forward.w1.scales": "model.safetensors",
         "model.layers.2.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.2.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.2.feed_forward.w2.scales": "model.safetensors",
         "model.layers.2.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.2.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.2.feed_forward.w3.scales": "model.safetensors",
         "model.layers.2.feed_forward.w3.weight": "model.safetensors",
         "model.layers.2.ffn_norm.weight": "model.safetensors",
         "model.layers.2.operator_norm.weight": "model.safetensors",
         "model.layers.2.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.2.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.2.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.2.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.2.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.2.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.2.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.2.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.2.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.2.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.2.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.2.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.2.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.2.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.3.conv.conv.weight": "model.safetensors",
-        "model.layers.3.conv.in_proj.biases": "model.safetensors",
-        "model.layers.3.conv.in_proj.scales": "model.safetensors",
         "model.layers.3.conv.in_proj.weight": "model.safetensors",
-        "model.layers.3.conv.out_proj.biases": "model.safetensors",
-        "model.layers.3.conv.out_proj.scales": "model.safetensors",
         "model.layers.3.conv.out_proj.weight": "model.safetensors",
-        "model.layers.3.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.3.feed_forward.w1.scales": "model.safetensors",
         "model.layers.3.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.3.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.3.feed_forward.w2.scales": "model.safetensors",
         "model.layers.3.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.3.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.3.feed_forward.w3.scales": "model.safetensors",
         "model.layers.3.feed_forward.w3.weight": "model.safetensors",
         "model.layers.3.ffn_norm.weight": "model.safetensors",
         "model.layers.3.operator_norm.weight": "model.safetensors",
         "model.layers.4.conv.conv.weight": "model.safetensors",
-        "model.layers.4.conv.in_proj.biases": "model.safetensors",
-        "model.layers.4.conv.in_proj.scales": "model.safetensors",
         "model.layers.4.conv.in_proj.weight": "model.safetensors",
-        "model.layers.4.conv.out_proj.biases": "model.safetensors",
-        "model.layers.4.conv.out_proj.scales": "model.safetensors",
         "model.layers.4.conv.out_proj.weight": "model.safetensors",
-        "model.layers.4.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.4.feed_forward.w1.scales": "model.safetensors",
         "model.layers.4.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.4.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.4.feed_forward.w2.scales": "model.safetensors",
         "model.layers.4.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.4.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.4.feed_forward.w3.scales": "model.safetensors",
         "model.layers.4.feed_forward.w3.weight": "model.safetensors",
         "model.layers.4.ffn_norm.weight": "model.safetensors",
         "model.layers.4.operator_norm.weight": "model.safetensors",
-        "model.layers.5.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.5.feed_forward.w1.scales": "model.safetensors",
         "model.layers.5.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.5.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.5.feed_forward.w2.scales": "model.safetensors",
         "model.layers.5.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.5.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.5.feed_forward.w3.scales": "model.safetensors",
         "model.layers.5.feed_forward.w3.weight": "model.safetensors",
         "model.layers.5.ffn_norm.weight": "model.safetensors",
         "model.layers.5.operator_norm.weight": "model.safetensors",
         "model.layers.5.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.5.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.5.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.5.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.5.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.5.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.5.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.5.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.5.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.5.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.5.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.5.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.5.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.5.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.6.conv.conv.weight": "model.safetensors",
-        "model.layers.6.conv.in_proj.biases": "model.safetensors",
-        "model.layers.6.conv.in_proj.scales": "model.safetensors",
         "model.layers.6.conv.in_proj.weight": "model.safetensors",
-        "model.layers.6.conv.out_proj.biases": "model.safetensors",
-        "model.layers.6.conv.out_proj.scales": "model.safetensors",
         "model.layers.6.conv.out_proj.weight": "model.safetensors",
-        "model.layers.6.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.6.feed_forward.w1.scales": "model.safetensors",
         "model.layers.6.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.6.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.6.feed_forward.w2.scales": "model.safetensors",
         "model.layers.6.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.6.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.6.feed_forward.w3.scales": "model.safetensors",
         "model.layers.6.feed_forward.w3.weight": "model.safetensors",
         "model.layers.6.ffn_norm.weight": "model.safetensors",
         "model.layers.6.operator_norm.weight": "model.safetensors",
         "model.layers.7.conv.conv.weight": "model.safetensors",
-        "model.layers.7.conv.in_proj.biases": "model.safetensors",
-        "model.layers.7.conv.in_proj.scales": "model.safetensors",
         "model.layers.7.conv.in_proj.weight": "model.safetensors",
-        "model.layers.7.conv.out_proj.biases": "model.safetensors",
-        "model.layers.7.conv.out_proj.scales": "model.safetensors",
         "model.layers.7.conv.out_proj.weight": "model.safetensors",
-        "model.layers.7.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.7.feed_forward.w1.scales": "model.safetensors",
         "model.layers.7.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.7.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.7.feed_forward.w2.scales": "model.safetensors",
         "model.layers.7.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.7.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.7.feed_forward.w3.scales": "model.safetensors",
         "model.layers.7.feed_forward.w3.weight": "model.safetensors",
         "model.layers.7.ffn_norm.weight": "model.safetensors",
         "model.layers.7.operator_norm.weight": "model.safetensors",
-        "model.layers.8.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.8.feed_forward.w1.scales": "model.safetensors",
         "model.layers.8.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.8.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.8.feed_forward.w2.scales": "model.safetensors",
         "model.layers.8.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.8.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.8.feed_forward.w3.scales": "model.safetensors",
         "model.layers.8.feed_forward.w3.weight": "model.safetensors",
         "model.layers.8.ffn_norm.weight": "model.safetensors",
         "model.layers.8.operator_norm.weight": "model.safetensors",
         "model.layers.8.self_attn.k_layernorm.weight": "model.safetensors",
-        "model.layers.8.self_attn.k_proj.biases": "model.safetensors",
-        "model.layers.8.self_attn.k_proj.scales": "model.safetensors",
         "model.layers.8.self_attn.k_proj.weight": "model.safetensors",
-        "model.layers.8.self_attn.out_proj.biases": "model.safetensors",
-        "model.layers.8.self_attn.out_proj.scales": "model.safetensors",
         "model.layers.8.self_attn.out_proj.weight": "model.safetensors",
         "model.layers.8.self_attn.q_layernorm.weight": "model.safetensors",
-        "model.layers.8.self_attn.q_proj.biases": "model.safetensors",
-        "model.layers.8.self_attn.q_proj.scales": "model.safetensors",
         "model.layers.8.self_attn.q_proj.weight": "model.safetensors",
-        "model.layers.8.self_attn.v_proj.biases": "model.safetensors",
-        "model.layers.8.self_attn.v_proj.scales": "model.safetensors",
         "model.layers.8.self_attn.v_proj.weight": "model.safetensors",
         "model.layers.9.conv.conv.weight": "model.safetensors",
-        "model.layers.9.conv.in_proj.biases": "model.safetensors",
-        "model.layers.9.conv.in_proj.scales": "model.safetensors",
         "model.layers.9.conv.in_proj.weight": "model.safetensors",
-        "model.layers.9.conv.out_proj.biases": "model.safetensors",
-        "model.layers.9.conv.out_proj.scales": "model.safetensors",
         "model.layers.9.conv.out_proj.weight": "model.safetensors",
-        "model.layers.9.feed_forward.w1.biases": "model.safetensors",
-        "model.layers.9.feed_forward.w1.scales": "model.safetensors",
         "model.layers.9.feed_forward.w1.weight": "model.safetensors",
-        "model.layers.9.feed_forward.w2.biases": "model.safetensors",
-        "model.layers.9.feed_forward.w2.scales": "model.safetensors",
         "model.layers.9.feed_forward.w2.weight": "model.safetensors",
-        "model.layers.9.feed_forward.w3.biases": "model.safetensors",
-        "model.layers.9.feed_forward.w3.scales": "model.safetensors",
         "model.layers.9.feed_forward.w3.weight": "model.safetensors",
         "model.layers.9.ffn_norm.weight": "model.safetensors",
         "model.layers.9.operator_norm.weight": "model.safetensors"