Add files using upload-large-folder tool

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-Instruct-250M",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41673fb85f448ff15356e4dfcf9e039d7c6e3cffa1936bc3513f55453005e63e
+size 39135840

added_tokens.json

@@ -0,0 +1,130 @@
+{
+  "<end_of_utterance>": 49279,
+  "<fake_token_around_image>": 49189,
+  "<global-img>": 49152,
+  "<image>": 49190,
+  "<row_1_col_1>": 49153,
+  "<row_1_col_2>": 49154,
+  "<row_1_col_3>": 49155,
+  "<row_1_col_4>": 49156,
+  "<row_1_col_5>": 49157,
+  "<row_1_col_6>": 49158,
+  "<row_2_col_1>": 49159,
+  "<row_2_col_2>": 49160,
+  "<row_2_col_3>": 49161,
+  "<row_2_col_4>": 49162,
+  "<row_2_col_5>": 49163,
+  "<row_2_col_6>": 49164,
+  "<row_3_col_1>": 49165,
+  "<row_3_col_2>": 49166,
+  "<row_3_col_3>": 49167,
+  "<row_3_col_4>": 49168,
+  "<row_3_col_5>": 49169,
+  "<row_3_col_6>": 49170,
+  "<row_4_col_1>": 49171,
+  "<row_4_col_2>": 49172,
+  "<row_4_col_3>": 49173,
+  "<row_4_col_4>": 49174,
+  "<row_4_col_5>": 49175,
+  "<row_4_col_6>": 49176,
+  "<row_5_col_1>": 49177,
+  "<row_5_col_2>": 49178,
+  "<row_5_col_3>": 49179,
+  "<row_5_col_4>": 49180,
+  "<row_5_col_5>": 49181,
+  "<row_5_col_6>": 49182,
+  "<row_6_col_1>": 49183,
+  "<row_6_col_2>": 49184,
+  "<row_6_col_3>": 49185,
+  "<row_6_col_4>": 49186,
+  "<row_6_col_5>": 49187,
+  "<row_6_col_6>": 49188,
+  "<|reserved_special_token_0|>": 49191,
+  "<|reserved_special_token_10|>": 49201,
+  "<|reserved_special_token_11|>": 49202,
+  "<|reserved_special_token_12|>": 49203,
+  "<|reserved_special_token_13|>": 49204,
+  "<|reserved_special_token_14|>": 49205,
+  "<|reserved_special_token_15|>": 49206,
+  "<|reserved_special_token_16|>": 49207,
+  "<|reserved_special_token_17|>": 49208,
+  "<|reserved_special_token_18|>": 49209,
+  "<|reserved_special_token_19|>": 49210,
+  "<|reserved_special_token_1|>": 49192,
+  "<|reserved_special_token_20|>": 49211,
+  "<|reserved_special_token_21|>": 49212,
+  "<|reserved_special_token_22|>": 49213,
+  "<|reserved_special_token_23|>": 49214,
+  "<|reserved_special_token_24|>": 49215,
+  "<|reserved_special_token_25|>": 49216,
+  "<|reserved_special_token_26|>": 49217,
+  "<|reserved_special_token_27|>": 49218,
+  "<|reserved_special_token_28|>": 49219,
+  "<|reserved_special_token_29|>": 49220,
+  "<|reserved_special_token_2|>": 49193,
+  "<|reserved_special_token_30|>": 49221,
+  "<|reserved_special_token_31|>": 49222,
+  "<|reserved_special_token_32|>": 49223,
+  "<|reserved_special_token_33|>": 49224,
+  "<|reserved_special_token_34|>": 49225,
+  "<|reserved_special_token_35|>": 49226,
+  "<|reserved_special_token_36|>": 49227,
+  "<|reserved_special_token_37|>": 49228,
+  "<|reserved_special_token_38|>": 49229,
+  "<|reserved_special_token_39|>": 49230,
+  "<|reserved_special_token_3|>": 49194,
+  "<|reserved_special_token_40|>": 49231,
+  "<|reserved_special_token_41|>": 49232,
+  "<|reserved_special_token_42|>": 49233,
+  "<|reserved_special_token_43|>": 49234,
+  "<|reserved_special_token_44|>": 49235,
+  "<|reserved_special_token_45|>": 49236,
+  "<|reserved_special_token_46|>": 49237,
+  "<|reserved_special_token_47|>": 49238,
+  "<|reserved_special_token_48|>": 49239,
+  "<|reserved_special_token_49|>": 49240,
+  "<|reserved_special_token_4|>": 49195,
+  "<|reserved_special_token_50|>": 49241,
+  "<|reserved_special_token_51|>": 49242,
+  "<|reserved_special_token_52|>": 49243,
+  "<|reserved_special_token_53|>": 49244,
+  "<|reserved_special_token_54|>": 49245,
+  "<|reserved_special_token_55|>": 49246,
+  "<|reserved_special_token_56|>": 49247,
+  "<|reserved_special_token_57|>": 49248,
+  "<|reserved_special_token_58|>": 49249,
+  "<|reserved_special_token_59|>": 49250,
+  "<|reserved_special_token_5|>": 49196,
+  "<|reserved_special_token_60|>": 49251,
+  "<|reserved_special_token_61|>": 49252,
+  "<|reserved_special_token_62|>": 49253,
+  "<|reserved_special_token_63|>": 49254,
+  "<|reserved_special_token_64|>": 49255,
+  "<|reserved_special_token_65|>": 49256,
+  "<|reserved_special_token_66|>": 49257,
+  "<|reserved_special_token_67|>": 49258,
+  "<|reserved_special_token_68|>": 49259,
+  "<|reserved_special_token_69|>": 49260,
+  "<|reserved_special_token_6|>": 49197,
+  "<|reserved_special_token_70|>": 49261,
+  "<|reserved_special_token_71|>": 49262,
+  "<|reserved_special_token_72|>": 49263,
+  "<|reserved_special_token_73|>": 49264,
+  "<|reserved_special_token_74|>": 49265,
+  "<|reserved_special_token_75|>": 49266,
+  "<|reserved_special_token_76|>": 49267,
+  "<|reserved_special_token_77|>": 49268,
+  "<|reserved_special_token_78|>": 49269,
+  "<|reserved_special_token_79|>": 49270,
+  "<|reserved_special_token_7|>": 49198,
+  "<|reserved_special_token_80|>": 49271,
+  "<|reserved_special_token_81|>": 49272,
+  "<|reserved_special_token_82|>": 49273,
+  "<|reserved_special_token_83|>": 49274,
+  "<|reserved_special_token_84|>": 49275,
+  "<|reserved_special_token_85|>": 49276,
+  "<|reserved_special_token_86|>": 49277,
+  "<|reserved_special_token_87|>": 49278,
+  "<|reserved_special_token_8|>": 49199,
+  "<|reserved_special_token_9|>": 49200
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"
+}

checkpoint-11082/README.md

@@ -0,0 +1,202 @@
+---
+base_model: ./models/ColSmolVLM-Instruct-250M
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

checkpoint-11082/adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-Instruct-250M",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-11082/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41673fb85f448ff15356e4dfcf9e039d7c6e3cffa1936bc3513f55453005e63e
+size 39135840

checkpoint-11082/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebcf9c079f09e188271298267be47877e242a46d3d509a54605a1d61a6e183bd
+size 78504446

checkpoint-11082/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c653672b3faef730c291bb1cb7ca4e4761c3c7e2114dfe4db070e1dd92e7d15
+size 14244

checkpoint-11082/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6492f98570ecb136d3a3c030cd8d38f5d102dfae6ace1f3bfc893278748c61b
+size 1064

checkpoint-11082/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d30e8edca1c2ae01e39ae490eabdb65ebc3e2c9197956089ece833734f6e7c7
+size 5304

config.json

@@ -0,0 +1,266 @@
+{
+  "_name_or_path": "models/SmolVLM-Instruct-250M",
+  "architectures": [
+    "ColIdefics3"
+  ],
+  "image_token_id": 49190,
+  "model_type": "idefics3",
+  "scale_factor": 4,
+  "text_config": {
+    "_attn_implementation_autoset": false,
+    "_flash_attn_2_enabled": true,
+    "_name_or_path": "None",
+    "add_cross_attention": false,
+    "architectures": [
+      "VLlama3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 576,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.041666666666666664,
+    "intermediate_size": 1536,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "is_llama_config": true,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 8192,
+    "min_length": 0,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "neftune_noise_alpha": 0.0,
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 9,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 30,
+    "num_key_value_heads": 3,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "perceiver_config": {
+      "_attn_implementation_autoset": false,
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "silu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "vllama3",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_key_value_heads": 1,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "qk_layer_norms_perceiver": false,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "resampler_depth": 6,
+      "resampler_head_dim": 96,
+      "resampler_n_heads": 16,
+      "resampler_n_latents": 64,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "transformers_version": "4.46.0",
+      "typical_p": 1.0,
+      "use_bfloat16": false
+    },
+    "pixel_shuffle_factor": 4,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_layer_norms": false,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_interleaved": false,
+    "rope_scaling": null,
+    "rope_theta": 100000,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers.js_config": {
+      "kv_cache_dtype": {
+        "fp16": "float16",
+        "q4f16": "float16"
+      }
+    },
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_resampler": false,
+    "vocab_size": 49280
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vision_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_image_size": {
+      "longest_edge": 512
+    },
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "idefics3",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "size": {
+      "longest_edge": 2048
+    },
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_base_siglip": true,
+    "use_bfloat16": false
+  },
+  "vocab_size": 49280
+}

dump/tmp_adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-Instruct-250M",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

dump/tmp_adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5177b81908576831837acf5ce33911d21fd9dd47a091f632b864afe1b9a0d9a5
+size 19598104

git_hash.txt

@@ -0,0 +1 @@
+8b4f75e476bddc6c34a02722761bd3ada6ac0d3d

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d026bad7c9d27edaeaf8ba4f2f0cd28406179819e06fa52693c051838c9069
+size 456405944

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "do_convert_rgb": true,
+  "do_image_splitting": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Idefics3ImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_image_size": {
+    "longest_edge": 512
+  },
+  "processor_class": "ColIdefics3Processor",
+  "resample": 1,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 2048
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_len": 64,
+  "processor_class": "Idefics3Processor"
+}

results.json

@@ -0,0 +1 @@
+{"./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.664, "ndcg_at_3": 0.71728, "ndcg_at_5": 0.73063, "ndcg_at_10": 0.75071, "ndcg_at_20": 0.76029, "ndcg_at_50": 0.77029, "ndcg_at_100": 0.77647, "map_at_1": 0.664, "map_at_3": 0.70333, "map_at_5": 0.71083, "map_at_10": 0.71913, "map_at_20": 0.72174, "map_at_50": 0.72338, "map_at_100": 0.72393, "recall_at_1": 0.664, "recall_at_3": 0.758, "recall_at_5": 0.79, "recall_at_10": 0.852, "recall_at_20": 0.89, "recall_at_50": 0.94, "recall_at_100": 0.978, "precision_at_1": 0.664, "precision_at_3": 0.25267, "precision_at_5": 0.158, "precision_at_10": 0.0852, "precision_at_20": 0.0445, "precision_at_50": 0.0188, "precision_at_100": 0.00978, "mrr_at_1": 0.666, "mrr_at_3": 0.7036666666666667, "mrr_at_5": 0.7110666666666667, "mrr_at_10": 0.719484126984127, "mrr_at_20": 0.722312611975228, "mrr_at_50": 0.7237972836422734, "mrr_at_100": 0.7243297450156544, "naucs_at_1_max": 0.5889213023056643, "naucs_at_1_std": -0.03738843019443627, "naucs_at_1_diff1": 0.9044815265827187, "naucs_at_3_max": 0.6597876116843661, "naucs_at_3_std": 0.041596651295516755, "naucs_at_3_diff1": 0.8645218684327024, "naucs_at_5_max": 0.6839060283261598, "naucs_at_5_std": 0.11626710270036247, "naucs_at_5_diff1": 0.8423112992040783, "naucs_at_10_max": 0.7812541175850534, "naucs_at_10_std": 0.21647297546578018, "naucs_at_10_diff1": 0.8277202413893048, "naucs_at_20_max": 0.7649600549780963, "naucs_at_20_std": 0.12761790224207298, "naucs_at_20_diff1": 0.7988574864702351, "naucs_at_50_max": 0.7467787114845937, "naucs_at_50_std": 0.2255835667600402, "naucs_at_50_diff1": 0.7687519452225302, "naucs_at_100_max": 0.8461505814447046, "naucs_at_100_std": 0.3166539343009966, "naucs_at_100_diff1": 0.8011628893981787}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.4745, "ndcg_at_3": 0.54173, "ndcg_at_5": 0.56769, "ndcg_at_10": 0.5861, "ndcg_at_20": 0.6048, "ndcg_at_50": 0.61661, "ndcg_at_100": 0.62385, "map_at_1": 0.4745, "map_at_3": 0.52513, "map_at_5": 0.53976, "map_at_10": 0.54722, "map_at_20": 0.55245, "map_at_50": 0.55432, "map_at_100": 0.55497, "recall_at_1": 0.4745, "recall_at_3": 0.5898, "recall_at_5": 0.65188, "recall_at_10": 0.70953, "recall_at_20": 0.78271, "recall_at_50": 0.84257, "recall_at_100": 0.88692, "precision_at_1": 0.4745, "precision_at_3": 0.1966, "precision_at_5": 0.13038, "precision_at_10": 0.07095, "precision_at_20": 0.03914, "precision_at_50": 0.01685, "precision_at_100": 0.00887, "mrr_at_1": 0.4722838137472284, "mrr_at_3": 0.5240206947524021, "mrr_at_5": 0.5384331116038433, "mrr_at_10": 0.545998310632457, "mrr_at_20": 0.5506564713842315, "mrr_at_50": 0.5529029054852187, "mrr_at_100": 0.5535124417758647, "naucs_at_1_max": 0.24308198716486704, "naucs_at_1_std": 0.4263216056628872, "naucs_at_1_diff1": 0.8579772747628749, "naucs_at_3_max": 0.16805600057176534, "naucs_at_3_std": 0.3924645370055808, "naucs_at_3_diff1": 0.7770233728520576, "naucs_at_5_max": 0.1218197178734784, "naucs_at_5_std": 0.4817802245485417, "naucs_at_5_diff1": 0.7532614461045579, "naucs_at_10_max": 0.03428092836449581, "naucs_at_10_std": 0.5151585952399194, "naucs_at_10_diff1": 0.7143986620927637, "naucs_at_20_max": -0.13960806153123748, "naucs_at_20_std": 0.6261345359875006, "naucs_at_20_diff1": 0.6826006385134176, "naucs_at_50_max": -0.2400541568650652, "naucs_at_50_std": 0.7530843562467058, "naucs_at_50_diff1": 0.6593166140032869, "naucs_at_100_max": -0.08884413440903777, "naucs_at_100_std": 0.7993547173030003, "naucs_at_100_diff1": 0.6944870252918233}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.91, "ndcg_at_3": 0.93262, "ndcg_at_5": 0.93693, "ndcg_at_10": 0.93994, "ndcg_at_20": 0.94494, "ndcg_at_50": 0.94702, "ndcg_at_100": 0.94702, "map_at_1": 0.91, "map_at_3": 0.92667, "map_at_5": 0.92917, "map_at_10": 0.93028, "map_at_20": 0.93161, "map_at_50": 0.93198, "map_at_100": 0.93198, "recall_at_1": 0.91, "recall_at_3": 0.95, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.31667, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9283333333333332, "mrr_at_5": 0.9308333333333333, "mrr_at_10": 0.9319444444444445, "mrr_at_20": 0.9332777777777779, "mrr_at_50": 0.9336944444444444, "mrr_at_100": 0.9336944444444444, "naucs_at_1_max": 0.3011723207801644, "naucs_at_1_std": -0.4870837223778408, "naucs_at_1_diff1": 0.8742089428363928, "naucs_at_3_max": 0.730158730158726, "naucs_at_3_std": -0.21027077497665203, "naucs_at_3_diff1": 0.9183006535947692, "naucs_at_5_max": 0.6953781512605006, "naucs_at_5_std": -0.48015873015872135, "naucs_at_5_diff1": 0.8978758169934612, "naucs_at_10_max": 0.807812013694364, "naucs_at_10_std": -0.06022408963585601, "naucs_at_10_diff1": 0.8638344226579531, "naucs_at_20_max": 0.5541549953314738, "naucs_at_20_std": -0.1713352007469681, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.64095, "ndcg_at_3": 0.736, "ndcg_at_5": 0.7601, "ndcg_at_10": 0.7803, "ndcg_at_20": 0.78827, "ndcg_at_50": 0.79223, "ndcg_at_100": 0.7941, "map_at_1": 0.64095, "map_at_3": 0.71203, "map_at_5": 0.72546, "map_at_10": 0.734, "map_at_20": 0.73618, "map_at_50": 0.73687, "map_at_100": 0.73703, "recall_at_1": 0.64095, "recall_at_3": 0.80559, "recall_at_5": 0.86391, "recall_at_10": 0.92527, "recall_at_20": 0.95687, "recall_at_50": 0.97631, "recall_at_100": 0.98785, "precision_at_1": 0.64095, "precision_at_3": 0.26853, "precision_at_5": 0.17278, "precision_at_10": 0.09253, "precision_at_20": 0.04784, "precision_at_50": 0.01953, "precision_at_100": 0.00988, "mrr_at_1": 0.6354799513973268, "mrr_at_3": 0.7106115836371001, "mrr_at_5": 0.7234305386796274, "mrr_at_10": 0.731879833747999, "mrr_at_20": 0.7340755638044026, "mrr_at_50": 0.7348432926756366, "mrr_at_100": 0.7350248567706289, "naucs_at_1_max": 0.19811506816235372, "naucs_at_1_std": -0.2554693200419591, "naucs_at_1_diff1": 0.7675648256745395, "naucs_at_3_max": 0.2318301363088243, "naucs_at_3_std": -0.27759660815315446, "naucs_at_3_diff1": 0.6712729518538324, "naucs_at_5_max": 0.24793757976069947, "naucs_at_5_std": -0.2400954177562814, "naucs_at_5_diff1": 0.6522915581849575, "naucs_at_10_max": 0.33999905443604334, "naucs_at_10_std": -0.09243929665705147, "naucs_at_10_diff1": 0.6186302038158574, "naucs_at_20_max": 0.33414218317326, "naucs_at_20_std": 0.10107766265773643, "naucs_at_20_diff1": 0.5589735061846622, "naucs_at_50_max": 0.39336569274053196, "naucs_at_50_std": 0.3146553201604068, "naucs_at_50_diff1": 0.5632664235073339, "naucs_at_100_max": 0.6769613453644149, "naucs_at_100_std": 0.7790878478446186, "naucs_at_100_diff1": 0.6726227816477444}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.7834, "ndcg_at_3": 0.83083, "ndcg_at_5": 0.84232, "ndcg_at_10": 0.85157, "ndcg_at_20": 0.85985, "ndcg_at_50": 0.86443, "ndcg_at_100": 0.8657, "map_at_1": 0.7834, "map_at_3": 0.81984, "map_at_5": 0.82611, "map_at_10": 0.82998, "map_at_20": 0.8323, "map_at_50": 0.83311, "map_at_100": 0.83321, "recall_at_1": 0.7834, "recall_at_3": 0.86235, "recall_at_5": 0.89069, "recall_at_10": 0.91903, "recall_at_20": 0.95142, "recall_at_50": 0.97368, "recall_at_100": 0.98178, "precision_at_1": 0.7834, "precision_at_3": 0.28745, "precision_at_5": 0.17814, "precision_at_10": 0.0919, "precision_at_20": 0.04757, "precision_at_50": 0.01947, "precision_at_100": 0.00982, "mrr_at_1": 0.7813765182186235, "mrr_at_3": 0.8188259109311741, "mrr_at_5": 0.8248987854251012, "mrr_at_10": 0.829565098644046, "mrr_at_20": 0.8313956960718094, "mrr_at_50": 0.8322900405575263, "mrr_at_100": 0.832391579382424, "naucs_at_1_max": 0.518447362653061, "naucs_at_1_std": 0.025991835344041193, "naucs_at_1_diff1": 0.9010494190422811, "naucs_at_3_max": 0.5691970896074877, "naucs_at_3_std": 0.020323756354481724, "naucs_at_3_diff1": 0.8408073942635276, "naucs_at_5_max": 0.5665846312895253, "naucs_at_5_std": 0.08497408554034125, "naucs_at_5_diff1": 0.82223001004187, "naucs_at_10_max": 0.6452921091747841, "naucs_at_10_std": 0.19809424135208908, "naucs_at_10_diff1": 0.8431957937117052, "naucs_at_20_max": 0.7447325753492595, "naucs_at_20_std": 0.4528568090604771, "naucs_at_20_diff1": 0.8030103604465574, "naucs_at_50_max": 0.8343240898820317, "naucs_at_50_std": 0.6770625322907705, "naucs_at_50_diff1": 0.8472448651285527, "naucs_at_100_max": 0.8102131093810145, "naucs_at_100_std": 0.7381222519798937, "naucs_at_100_diff1": 0.8742471393840519}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.96786, "ndcg_at_5": 0.96786, "ndcg_at_10": 0.96786, "ndcg_at_20": 0.96786, "ndcg_at_50": 0.96786, "ndcg_at_100": 0.96786, "map_at_1": 0.92, "map_at_3": 0.95667, "map_at_5": 0.95667, "map_at_10": 0.95667, "map_at_20": 0.95667, "map_at_50": 0.95667, "map_at_100": 0.95667, "recall_at_1": 0.92, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.9666666666666667, "mrr_at_5": 0.9666666666666667, "mrr_at_10": 0.9666666666666667, "mrr_at_20": 0.9666666666666667, "mrr_at_50": 0.9666666666666667, "mrr_at_100": 0.9666666666666667, "naucs_at_1_max": 0.7619047619047616, "naucs_at_1_std": 0.24060457516339795, "naucs_at_1_diff1": 0.9162581699346404, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.56786, "ndcg_at_3": 0.62806, "ndcg_at_5": 0.65635, "ndcg_at_10": 0.67295, "ndcg_at_20": 0.68721, "ndcg_at_50": 0.70267, "ndcg_at_100": 0.71485, "map_at_1": 0.56786, "map_at_3": 0.6131, "map_at_5": 0.62899, "map_at_10": 0.63542, "map_at_20": 0.63924, "map_at_50": 0.64167, "map_at_100": 0.64292, "recall_at_1": 0.56786, "recall_at_3": 0.67143, "recall_at_5": 0.73929, "recall_at_10": 0.79286, "recall_at_20": 0.85, "recall_at_50": 0.92857, "recall_at_100": 1.0, "precision_at_1": 0.56786, "precision_at_3": 0.22381, "precision_at_5": 0.14786, "precision_at_10": 0.07929, "precision_at_20": 0.0425, "precision_at_50": 0.01857, "precision_at_100": 0.01, "mrr_at_1": 0.5678571428571428, "mrr_at_3": 0.6113095238095237, "mrr_at_5": 0.6277380952380953, "mrr_at_10": 0.6339356575963718, "mrr_at_20": 0.6379618770431741, "mrr_at_50": 0.6403947405923297, "mrr_at_100": 0.6416450055922677, "naucs_at_1_max": 0.15868730409527165, "naucs_at_1_std": 0.005476384052524141, "naucs_at_1_diff1": 0.6898556504931816, "naucs_at_3_max": 0.2265285828624092, "naucs_at_3_std": 0.07120249326962727, "naucs_at_3_diff1": 0.6460005820862944, "naucs_at_5_max": 0.18104774909508717, "naucs_at_5_std": 0.049974351749500545, "naucs_at_5_diff1": 0.599728345290476, "naucs_at_10_max": 0.08937034529451629, "naucs_at_10_std": 0.021830460219087747, "naucs_at_10_diff1": 0.5156241862403002, "naucs_at_20_max": -0.015480451861837658, "naucs_at_20_std": -0.04558133048207932, "naucs_at_20_diff1": 0.5020338431500163, "naucs_at_50_max": 0.02670401493930801, "naucs_at_50_std": -0.1887955182072852, "naucs_at_50_diff1": 0.31573295985060545, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.88, "ndcg_at_3": 0.92786, "ndcg_at_5": 0.94421, "ndcg_at_10": 0.94421, "ndcg_at_20": 0.94421, "ndcg_at_50": 0.94421, "ndcg_at_100": 0.94421, "map_at_1": 0.88, "map_at_3": 0.91667, "map_at_5": 0.92567, "map_at_10": 0.92567, "map_at_20": 0.92567, "map_at_50": 0.92567, "map_at_100": 0.92567, "recall_at_1": 0.88, "recall_at_3": 0.96, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.32, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9423333333333334, "mrr_at_10": 0.9423333333333334, "mrr_at_20": 0.9423333333333334, "mrr_at_50": 0.9423333333333334, "mrr_at_100": 0.9423333333333334, "naucs_at_1_max": 0.45863824371619505, "naucs_at_1_std": 0.23079064587973264, "naucs_at_1_diff1": 0.8033725739739099, "naucs_at_3_max": 0.8068394024276336, "naucs_at_3_std": 0.5087535014005626, "naucs_at_3_diff1": 0.6038748832866443, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.45, "ndcg_at_3": 0.59464, "ndcg_at_5": 0.63982, "ndcg_at_10": 0.66889, "ndcg_at_20": 0.67709, "ndcg_at_50": 0.68517, "ndcg_at_100": 0.68853, "map_at_1": 0.45, "map_at_3": 0.55833, "map_at_5": 0.58333, "map_at_10": 0.59531, "map_at_20": 0.59789, "map_at_50": 0.59924, "map_at_100": 0.59957, "recall_at_1": 0.45, "recall_at_3": 0.7, "recall_at_5": 0.81, "recall_at_10": 0.9, "recall_at_20": 0.93, "recall_at_50": 0.97, "recall_at_100": 0.99, "precision_at_1": 0.45, "precision_at_3": 0.23333, "precision_at_5": 0.162, "precision_at_10": 0.09, "precision_at_20": 0.0465, "precision_at_50": 0.0194, "precision_at_100": 0.0099, "mrr_at_1": 0.47, "mrr_at_3": 0.5833333333333334, "mrr_at_5": 0.6018333333333333, "mrr_at_10": 0.615718253968254, "mrr_at_20": 0.6165515873015873, "mrr_at_50": 0.6179859646889059, "mrr_at_100": 0.6183226650256062, "naucs_at_1_max": 0.105925489425784, "naucs_at_1_std": -0.060066784521705045, "naucs_at_1_diff1": 0.49797682184246717, "naucs_at_3_max": 0.11120518825436876, "naucs_at_3_std": -0.10580075662042858, "naucs_at_3_diff1": 0.46674473067915695, "naucs_at_5_max": 0.03777044371103874, "naucs_at_5_std": -0.21048771543820866, "naucs_at_5_diff1": 0.38344072502488297, "naucs_at_10_max": 0.09103641456582774, "naucs_at_10_std": -0.14047619047618726, "naucs_at_10_diff1": 0.32413632119514585, "naucs_at_20_max": -0.0814992663732126, "naucs_at_20_std": -0.3702147525676927, "naucs_at_20_diff1": 0.34020274776577397, "naucs_at_50_max": -0.20401493930905526, "naucs_at_50_std": -0.5308123249299683, "naucs_at_50_diff1": 0.7860255213196357, "naucs_at_100_max": -0.5634920634920583, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 0.35807656395892007}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95786, "ndcg_at_5": 0.96172, "ndcg_at_10": 0.96172, "ndcg_at_20": 0.96172, "ndcg_at_50": 0.96172, "ndcg_at_100": 0.96332, "map_at_1": 0.92, "map_at_3": 0.95, "map_at_5": 0.952, "map_at_10": 0.952, "map_at_20": 0.952, "map_at_50": 0.952, "map_at_100": 0.95213, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.95, "mrr_at_5": 0.9525, "mrr_at_10": 0.9525, "mrr_at_20": 0.9525, "mrr_at_50": 0.9525, "mrr_at_100": 0.9526315789473684, "naucs_at_1_max": 0.749649859943977, "naucs_at_1_std": 0.25116713352007414, "naucs_at_1_diff1": 0.9279295051353874, "naucs_at_3_max": 0.8611111111111119, "naucs_at_3_std": 0.6790382819794457, "naucs_at_3_diff1": 0.7117180205415458, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": 0.35807656395891135, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.8692810457516413, "naucs_at_50_max": 0.7222222222222041, "naucs_at_50_std": 0.35807656395892007, "naucs_at_50_diff1": 0.8692810457516374, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}}

special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,1183 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<global-img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<row_1_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<row_1_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49155": {
+      "content": "<row_1_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49156": {
+      "content": "<row_1_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49157": {
+      "content": "<row_1_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49158": {
+      "content": "<row_1_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49159": {
+      "content": "<row_2_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49160": {
+      "content": "<row_2_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49161": {
+      "content": "<row_2_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49162": {
+      "content": "<row_2_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49163": {
+      "content": "<row_2_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49164": {
+      "content": "<row_2_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49165": {
+      "content": "<row_3_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49166": {
+      "content": "<row_3_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49167": {
+      "content": "<row_3_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49168": {
+      "content": "<row_3_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49169": {
+      "content": "<row_3_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49170": {
+      "content": "<row_3_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49171": {
+      "content": "<row_4_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49172": {
+      "content": "<row_4_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49173": {
+      "content": "<row_4_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49174": {
+      "content": "<row_4_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49175": {
+      "content": "<row_4_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49176": {
+      "content": "<row_4_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49177": {
+      "content": "<row_5_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49178": {
+      "content": "<row_5_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49179": {
+      "content": "<row_5_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49180": {
+      "content": "<row_5_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49181": {
+      "content": "<row_5_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49182": {
+      "content": "<row_5_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49183": {
+      "content": "<row_6_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49184": {
+      "content": "<row_6_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49185": {
+      "content": "<row_6_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49186": {
+      "content": "<row_6_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49187": {
+      "content": "<row_6_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49188": {
+      "content": "<row_6_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49189": {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49190": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49191": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49192": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49193": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49194": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49195": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49196": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49197": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49198": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49199": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49200": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49201": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49202": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49203": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49204": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49205": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49206": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49207": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49208": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49209": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49210": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49211": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49212": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49213": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49214": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49215": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49216": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49217": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49218": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49219": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49220": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49221": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49222": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49223": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49224": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49225": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49226": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49227": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49228": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49229": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49230": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49231": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49232": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49233": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49234": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49235": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49236": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49237": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49238": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49239": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49240": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49241": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49242": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49243": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49244": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49245": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49246": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49247": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49248": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49249": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49250": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49251": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49252": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49253": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49254": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49255": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49256": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49257": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49258": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49259": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49260": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49261": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49262": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49263": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49264": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49265": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49266": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49267": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49268": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49269": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49270": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49271": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49272": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49273": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49274": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49275": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49276": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49277": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49278": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49279": {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<fake_token_around_image>",
+    "<image>",
+    "<end_of_utterance>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "processor_class": "ColIdefics3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

training_config.yml

@@ -0,0 +1,72 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/ColSmolVLM-Instruct-250M
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3Processor
+    pretrained_model_name_or_path:  "./models/ColSmolVLM-Instruct-250M"
+    # num_image_tokens: 2048
+    # max_length: 50
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3
+    pretrained_model_name_or_path: "./models/ColSmolVLM-Instruct-250M"
+    torch_dtype:  !ext torch.bfloat16
+    # use_cache: false
+    attn_implementation: "flash_attention_2"
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  loss_func:
+    (): colpali_engine.loss.late_interaction_losses.ColbertPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 3
+    per_device_train_batch_size: 8
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # gradient_checkpointing: true
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 4
+    eval_strategy: "steps"
+    dataloader_num_workers: 6
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-4
+    save_total_limit: 1
+    resume_from_checkpoint: true
+    # optim: "paged_adamw_8bit"
+    # wandb logging
+    # wandb_project: "colqwen2"
+    # run_name: "colqwen2-ba32-nolora"
+    report_to: "wandb"
+
+
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+