Update fine-tuned adapter weights

README.md

@@ -19,6 +19,6 @@ Tiny LLM is a fine-tuned language model by Rahul Dhole, built on top of Qwen2.5-
 ## Training
 
 - **Method**: LoRA (r=8, alpha=32)
-- **Epochs**: 50
+- **Epochs**: 10
 - **Learning Rate**: 0.001
 - **Data**: data/dummy_train.jsonl

adapter_config.json

@@ -1,9 +1,12 @@
 {
+  "alora_invocation_tokens": null,
   "alpha_pattern": {},
+  "arrow_config": null,
   "auto_mapping": null,
   "base_model_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
   "bias": "none",
   "corda_config": null,
+  "ensure_weight_tying": false,
   "eva_config": null,
   "exclude_modules": null,
   "fan_in_fan_out": false,
@@ -20,6 +23,7 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
+  "peft_version": "0.18.1",
   "qalora_group_size": 16,
   "r": 8,
   "rank_pattern": {},

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7d417c1213718029a1c4cc61520bcaf45d48f39e0e983825ba1245978ac1879
+oid sha256:cfe9aab1757f20a25a8f7137e31901ced052a80c2a98858bd0cd10546184dd73
 size 2175168

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
-size 11421896
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -1,185 +1,11 @@
 {
-  "add_bos_token": false,
   "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<|object_ref_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151647": {
-      "content": "<|object_ref_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151648": {
-      "content": "<|box_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151649": {
-      "content": "<|box_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151650": {
-      "content": "<|quad_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151651": {
-      "content": "<|quad_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151652": {
-      "content": "<|vision_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151653": {
-      "content": "<|vision_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151654": {
-      "content": "<|vision_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151655": {
-      "content": "<|image_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151656": {
-      "content": "<|video_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151657": {
-      "content": "<tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151658": {
-      "content": "</tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151659": {
-      "content": "<|fim_prefix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151660": {
-      "content": "<|fim_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151661": {
-      "content": "<|fim_suffix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151662": {
-      "content": "<|fim_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151663": {
-      "content": "<|repo_name|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151664": {
-      "content": "<|file_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    }
-  },
-  "additional_special_tokens": [
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
     "<|object_ref_start|>",
@@ -194,11 +20,7 @@
     "<|image_pad|>",
     "<|video_pad|>"
   ],
-  "bos_token": null,
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "extra_special_tokens": {},
+  "is_local": false,
   "model_max_length": 131072,
   "pad_token": "<|endoftext|>",
   "split_special_tokens": false,

train_config.yaml

@@ -1,35 +1,30 @@
-model:
-  name: Qwen/Qwen2.5-0.5B-Instruct
-  dtype: float16
-
-# White-label metadata
-metadata:
-  model_name: Tiny LLM
-  author: Rahul Dhole
-  base_model: Qwen/Qwen2.5-0.5B-Instruct
-  license: apache-2.0
-  description: >
-    Tiny LLM is a fine-tuned language model by Rahul Dhole,
-    built on top of Qwen2.5-0.5B-Instruct using LoRA/PEFT.
-
+data:
+  path: data/dummy_train.jsonl
 lora:
-  r: 8
   alpha: 32
   dropout: 0.1
+  r: 8
   target_modules:
-    - q_proj
-    - v_proj
+  - q_proj
+  - v_proj
+metadata:
+  author: Rahul Dhole
+  base_model: Qwen/Qwen2.5-0.5B-Instruct
+  description: 'Tiny LLM is a fine-tuned language model by Rahul Dhole, built on top
+    of Qwen2.5-0.5B-Instruct using LoRA/PEFT.
 
+    '
+  license: apache-2.0
+  model_name: Tiny LLM
+model:
+  dtype: float32
+  name: Qwen/Qwen2.5-0.5B-Instruct
+output:
+  dir: outputs/qwen-fine-tuned
 training:
-  epochs: 50
   batch_size: 1
+  epochs: 10
   gradient_accumulation_steps: 4
   learning_rate: 0.001
-  max_length: 512
+  max_length: 256
   seed: 42
-
-data:
-  path: data/dummy_train.jsonl
-
-output:
-  dir: outputs/qwen-fine-tuned

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0d13eb4ff4f11267cc1f04ea4d7767ec2600a26e55bdf0620baf0a10c8d0b0e
-size 6225
+oid sha256:72969f98bd8768293347d4066f905a1c856b38761560138780eea84b87347511
+size 5585