Training in progress, epoch 1

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "Qwen/Qwen2-VL-7B-Instruct",
+  "base_model_name_or_path": "HuggingFaceTB/SmolVLM-Instruct",
   "bias": "none",
   "corda_config": null,
   "eva_config": null,
@@ -24,8 +24,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj"
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e3a9a961167e6590873dbf63089b55a005b5e0615c9e0e4aa182e9b55cd30e85
-size 10108960
+oid sha256:90f59e2520afbb03db1ad99f02d574bb5a261f690056468180b8190e90cbe79a
+size 10302960

added_tokens.json

@@ -1,16 +1,5 @@
 {
-  "<|box_end|>": 151649,
-  "<|box_start|>": 151648,
-  "<|endoftext|>": 151643,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644,
-  "<|image_pad|>": 151655,
-  "<|object_ref_end|>": 151647,
-  "<|object_ref_start|>": 151646,
-  "<|quad_end|>": 151651,
-  "<|quad_start|>": 151650,
-  "<|video_pad|>": 151656,
-  "<|vision_end|>": 151653,
-  "<|vision_pad|>": 151654,
-  "<|vision_start|>": 151652
+  "<end_of_utterance>": 49154,
+  "<fake_token_around_image>": 49152,
+  "<image>": 49153
 }

chat_template.jinja

@@ -1,7 +1,2 @@
-{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
-You are a helpful assistant.<|im_end|>
-{% endif %}<|im_start|>{{ message['role'] }}
-{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
-{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
-{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
-{% endif %}
+<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>
+{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}

runs/Jul29_15-43-44_ai3.ku.io/events.out.tfevents.1753778665.ai3.ku.io.3590973.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e874791c500f7cf69b6f3deb5e07b8ebe13e369fb5b08420e338578bfdc7af8d
-size 8603
+oid sha256:01c8f33c47a4fa155f490acf51059ff3c221e47d6a6fe25e505112ae26bda84d
+size 8810

runs/Jul29_16-42-15_ai3.ku.io/events.out.tfevents.1753782195.ai3.ku.io.3592098.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1ec1bc2b5b8fe18ced5f9fd861892bc671363c17cf5d4875c6ff52a4177c4a1
+size 9932

special_tokens_map.json

@@ -1,27 +1,49 @@
 {
   "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
   ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
-    "content": "<|im_end|>",
+    "content": "<end_of_utterance>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88a3a6fcb80132f76da8aa40cdc3fccd7e5d8468ef15421f5b0c2715e85217d2
-size 11420538
+oid sha256:43cdb6cd62df2c99a15307455f5990a7b579fefbd26c2624cd643bb7f43f4310
+size 3523394

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
-    "151643": {
+    "0": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -9,7 +9,7 @@
       "single_word": false,
       "special": true
     },
-    "151644": {
+    "1": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -17,7 +17,7 @@
       "single_word": false,
       "special": true
     },
-    "151645": {
+    "2": {
       "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
@@ -25,88 +25,136 @@
       "single_word": false,
       "special": true
     },
-    "151646": {
-      "content": "<|object_ref_start|>",
+    "3": {
+      "content": "<repo_name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151647": {
-      "content": "<|object_ref_end|>",
+    "4": {
+      "content": "<reponame>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151648": {
-      "content": "<|box_start|>",
+    "5": {
+      "content": "<file_sep>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151649": {
-      "content": "<|box_end|>",
+    "6": {
+      "content": "<filename>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151650": {
-      "content": "<|quad_start|>",
+    "7": {
+      "content": "<gh_stars>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151651": {
-      "content": "<|quad_end|>",
+    "8": {
+      "content": "<issue_start>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151652": {
-      "content": "<|vision_start|>",
+    "9": {
+      "content": "<issue_comment>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151653": {
-      "content": "<|vision_end|>",
+    "10": {
+      "content": "<issue_closed>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151654": {
-      "content": "<|vision_pad|>",
+    "11": {
+      "content": "<jupyter_start>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151655": {
-      "content": "<|image_pad|>",
+    "12": {
+      "content": "<jupyter_text>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "151656": {
-      "content": "<|video_pad|>",
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<end_of_utterance>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -115,29 +163,20 @@
     }
   },
   "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
+    "<fake_token_around_image>",
+    "<image>",
+    "<end_of_utterance>"
   ],
-  "bos_token": null,
+  "bos_token": "<|im_start|>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
+  "eos_token": "<end_of_utterance>",
   "extra_special_tokens": {},
-  "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "left",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "legacy": false,
+  "model_max_length": 16384,
+  "pad_token": "<|im_end|>",
+  "processor_class": "Idefics3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e37a39b2beabff1416eef7134bf6f8fde339ca2ea64fb41bfcb2dc6837e74f88
+oid sha256:fab74f982c315694a4543400e9618801b2cf892de4700dc8a940f07e84618bcd
 size 5624