wip

ROCO-idefics3.ipynb

@@ -22,7 +22,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {
     "executionInfo": {
      "elapsed": 2,
@@ -36,7 +36,23 @@
     },
     "id": "F-zJG-uPIy3d"
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "Exception",
+     "evalue": "You are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m----> 2\u001b[0m   \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcolab\u001b[39;00m\n\u001b[1;32m      3\u001b[0m   \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcolab\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m drive\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'google'",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m   drive\u001b[38;5;241m.\u001b[39mmount(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/content/drive\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mModuleNotFoundError\u001b[39;00m:\n\u001b[0;32m----> 7\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "\u001b[0;31mException\u001b[0m: You are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive"
+     ]
+    }
+   ],
    "source": [
     "try:\n",
     "  import google.colab\n",
@@ -56,7 +72,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {
     "executionInfo": {
      "elapsed": 1459,
@@ -103,27 +119,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
+     "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Token has not been saved to git credential helper.\n"
+      "Hugging Face token found in environment variable\n"
      ]
     },
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
-      "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
-      "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
-      "\n",
-      "git config --global credential.helper store\n",
-      "\n",
-      "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n"
+      "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.\n"
      ]
     }
    ],
@@ -228,7 +238,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -649,7 +659,245 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9371eea358754fe09d320c24e0ad42a6",
+       "model_id": "c681920ed6e24c35981eda639b1c4458",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "README.md:   0%|          | 0.00/4.50k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c5c781c8755e4ee79f5c972bb786ddda",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Resolving data files:   0%|          | 0/27 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4ece62cf670f441181bce789dbabb38d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Resolving data files:   0%|          | 0/27 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "368b3734b6094270874bc9ed05220b43",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading data:   0%|          | 0/27 [00:00<?, ?files/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e002292c8f73473ba28b353594d047a0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00000-of-00027.parquet:   0%|          | 0.00/497M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b6ec67734a7644c49c6d222db885d9fa",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00001-of-00027.parquet:   0%|          | 0.00/504M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d04c94230afb42c4b5488d1517c63fd6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00002-of-00027.parquet:   0%|          | 0.00/490M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0fd069cace5146b8be5b44a89dae32dd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00003-of-00027.parquet:   0%|          | 0.00/485M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e23e8d0ecb394513907c6ecb4a30783d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00004-of-00027.parquet:   0%|          | 0.00/510M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6d0a484c03274c628585bfd3938dd5dd",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00005-of-00027.parquet:   0%|          | 0.00/498M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8a9d84a4ba4c4a9d8762845b466159dc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00006-of-00027.parquet:   0%|          | 0.00/532M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ac978b853b8f4ce48b2240adad35813c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00007-of-00027.parquet:   0%|          | 0.00/482M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dc39d2085a8748269438f3f518a936fa",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00008-of-00027.parquet:   0%|          | 0.00/497M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "efd5dd9b5acf4a00b9fad256b239fe25",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00009-of-00027.parquet:   0%|          | 0.00/489M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0ee7a5cdf9fc44fb95a02a215db064b1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00010-of-00027.parquet:   0%|          | 0.00/484M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "d2bc061d040440d48bd1e068e7d0e754",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00011-of-00027.parquet:   0%|          | 0.00/508M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "290a81df45914de6865a1fd538746d8b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "train-00012-of-00027.parquet:   0%|          | 0.00/490M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e15f56550122473eaf0b9d2c33defc7a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -663,7 +911,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "739e7749e5364b63aa0dd766fe0db69b",
+       "model_id": "219eeafe2fde471d9c524a8f03884678",
        "version_major": 2,
        "version_minor": 0
       },
@@ -677,7 +925,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cf0aee0fbfed43788e5feebe465de1db",
+       "model_id": "afb3a6d52a6e46268803195d3f7e0e8e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -691,7 +939,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cfaca0fba2b94d1f992effbd930296c7",
+       "model_id": "fe952f2809c24da48e85d56b8de828d9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -705,7 +953,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a0c12b32b24a4b878c666755463d97ca",
+       "model_id": "09e7e2783b464c6197bebcbe45364e65",
        "version_major": 2,
        "version_minor": 0
       },
@@ -719,7 +967,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8b455932b90e426b8356094b7465670e",
+       "model_id": "f2142baf4df04c2e8fac7fda5cc4a4b4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -733,7 +981,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "54a181bd1f104a2f8d1fdee8495a546d",
+       "model_id": "ce16d5ccf5a244279e29b26f6b9f159f",
        "version_major": 2,
        "version_minor": 0
       },
@@ -747,7 +995,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b8886eba98a540bc8e6fa675a0b181c6",
+       "model_id": "d53b88d0ad5049e493578571a0d33740",
        "version_major": 2,
        "version_minor": 0
       },
@@ -761,7 +1009,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "49cb40f2e63f44198927441543a17eb6",
+       "model_id": "785e664dba3a40edb3858acccf6a07b0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -775,7 +1023,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "31e1d0b1553946959846dfc5a098f3c7",
+       "model_id": "ab300b08c57247d5a972d0d77acfddf4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -789,7 +1037,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "791f41cba3024fc2910974d8f1466c57",
+       "model_id": "00ab18dd02f34aa3a658b456d8bfe390",
        "version_major": 2,
        "version_minor": 0
       },
@@ -803,7 +1051,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e66e26e55af2424c8e262df2af686bba",
+       "model_id": "421d90d57c8e44a48479eef2eb40a479",
        "version_major": 2,
        "version_minor": 0
       },
@@ -817,7 +1065,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "70a9c0769e2f4d1295cd8fbc75f59000",
+       "model_id": "60abcaa9df3d43a99ef1e07e9b7fbe11",
        "version_major": 2,
        "version_minor": 0
       },
@@ -831,7 +1079,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a11e12a045c4469c81bd18dcf521e377",
+       "model_id": "79af8edddbfe4543a00e47d5f697866d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -845,7 +1093,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f8f7e87560fb4db0a73a1c0abca661a8",
+       "model_id": "9d14e22b823d4c9e85b17e4dbd57fec6",
        "version_major": 2,
        "version_minor": 0
       },
@@ -859,7 +1107,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6ab2e09c185e42af9667e6149ca44031",
+       "model_id": "ee5ba070e76b4854aadfac59b0237fbf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -873,7 +1121,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "72dea4c919dd4b8d86538228f6291213",
+       "model_id": "8ce96862fb234f6ea335071fc8114574",
        "version_major": 2,
        "version_minor": 0
       },
@@ -887,7 +1135,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ef3fbc6f624f44e1b3d01b33ecbd0e5f",
+       "model_id": "dd5860de108f49e5bcc609bb11a646df",
        "version_major": 2,
        "version_minor": 0
       },
@@ -901,7 +1149,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4bbdc7b9c1ca49cdba5c47b6ba9d5a8e",
+       "model_id": "bef0c258e4764f0ab9406467f01752a9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -915,7 +1163,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "479506696f244ce6b0c585e7da202f1c",
+       "model_id": "6c4d53689dad43dd8db780522139f599",
        "version_major": 2,
        "version_minor": 0
       },
@@ -929,7 +1177,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2870d5ecfed045a18c4e93eeb499da52",
+       "model_id": "d50075ea0fcd4a34ac88ed121b1e90bf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -943,7 +1191,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0435bb18b438409bb003553a9cab29d1",
+       "model_id": "a4e15de7a1c54400a1ebff02d4caa657",
        "version_major": 2,
        "version_minor": 0
       },
@@ -957,7 +1205,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4302f869cd2f4e8ab88751e2772b374b",
+       "model_id": "41fc8cb29962483e81fa8a640f633d46",
        "version_major": 2,
        "version_minor": 0
       },
@@ -971,7 +1219,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7e912d183c534543bba1e9aa40ce1c27",
+       "model_id": "388b29f8655b4fee87db053d591bd72d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -985,7 +1233,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b5c2227341f94d7483756b410497c6c6",
+       "model_id": "6c75919875544db98ee2c64215ecff97",
        "version_major": 2,
        "version_minor": 0
       },
@@ -999,7 +1247,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "91428e23d51f403e8eba0f05443bafba",
+       "model_id": "a4e8b6ecaf734f33a842f60681e23108",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1013,7 +1261,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cadd28d8df074bc79968fe53489d94d1",
+       "model_id": "7e497cfcd6f04a94aa0ac40d6df938b1",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1027,7 +1275,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "55263a9a632b42a6af60995cf89441c4",
+       "model_id": "56cde5f680f643be887d6f9f804676c9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1041,7 +1289,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b56539a9023f4806ba7374f6a3e01731",
+       "model_id": "fbcf378be86c40cdbf606a2446d1a252",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1055,7 +1303,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "321ad27da22a48078d2b16f646f0a2f8",
+       "model_id": "963491aad1f945cfb5811d7239880b54",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1086,7 +1334,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -1114,7 +1362,7 @@
        " 'cui': ['C0037005']}"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1125,7 +1373,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1154,7 +1402,7 @@
        "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=1684x2294>"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1174,7 +1422,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1217,7 +1465,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0915ca58b5974a558d9a97e6e5b49601",
+       "model_id": "f8115bd39ceb47678208bc6dc80a179a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1231,7 +1479,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6ddf29177eda49eaa001b66c4406d91d",
+       "model_id": "195ef910bea946cbac27ae80d12ab37d",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1245,7 +1493,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "81e9fd88f02b44a0bd0d361891a97b8d",
+       "model_id": "ee41668130414ec29bd670a4c00ea9dd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1259,7 +1507,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3731fd9db78c4e7bb71fcfabce3b5cf2",
+       "model_id": "cae512d46a20437cb2b9054e6d796129",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1273,7 +1521,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "892c80cdcfba41d4b7b7cac77c641a80",
+       "model_id": "15b4f6ea701b40f984b45da569b8fd50",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1287,7 +1535,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2e4ba2c07973429492728624aa9f6676",
+       "model_id": "711ead0dc9ba4017a5755a1cc111d8c3",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1301,7 +1549,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c9b31d5ca2d0424fa449f7301e0ec502",
+       "model_id": "009a9836643d4a1285ba017cb6dee9fb",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1322,7 +1570,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d407949500224df2ad24fb1a6b9ed96a",
+       "model_id": "1078bb163ca14f4cb242b3afb81b6a70",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1336,7 +1584,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6f77cab1ed284f9097605b8f43f12755",
+       "model_id": "813c7b8d26224b07893f652c1ab25acf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1350,7 +1598,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1495b59787e745928dc1512693389ed4",
+       "model_id": "8f312307a8c34e9ebac5a4006cb75b15",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1364,7 +1612,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "bb657ddcc31e4d72b04b5ed8049ece8c",
+       "model_id": "6805d6a27b4b404594f644d8289e3e0c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1378,7 +1626,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1bd3d44ebe444cf6aa57757d04f5d4a5",
+       "model_id": "dfa588033e244ba0aef4875d4dca0087",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1392,7 +1640,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8eb46982436d419fafd125d9ecabde87",
+       "model_id": "de1b8cc27c5744b4b34b48e7a1fb7a00",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1406,7 +1654,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ab006c8fef8642feb0c34d0710a9900f",
+       "model_id": "79b23b4c9373457fb28a1cdcc1b23277",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1420,7 +1668,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d6c8bc77933a44c4bf037f6541b6315b",
+       "model_id": "02c68a9944a44dcc882dcaa7722dfa9b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1715,6 +1963,57 @@
    "source": [
     "trainer.train()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "1694c769eb91432a90f7e6a69bfc8367",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "ename": "OutOfMemoryError",
+     "evalue": "CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 6.00 GiB of which 0 bytes is free. Of the allocated memory 20.39 GiB is allocated by PyTorch, and 155.53 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model \u001b[38;5;241m=\u001b[39m Idefics3ForConditionalGeneration\u001b[38;5;241m.\u001b[39mfrom_pretrained(source_model_id , torch_dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mbfloat16)\u001b[38;5;241m.\u001b[39mto(DEVICE)\n\u001b[1;32m      2\u001b[0m model\u001b[38;5;241m.\u001b[39mload_adapter(destination_model_id, device_map\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/modeling_utils.py:3167\u001b[0m, in \u001b[0;36mPreTrainedModel.to\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3162\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m dtype_present_in_args:\n\u001b[1;32m   3163\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   3164\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou cannot cast a GPTQ model in a new `dtype`. Make sure to load the model using `from_pretrained` using the desired\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3165\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m `dtype` by passing the correct `torch_dtype` argument.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   3166\u001b[0m         )\n\u001b[0;32m-> 3167\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1340\u001b[0m, in \u001b[0;36mModule.to\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1337\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1338\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[0;32m-> 1340\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_apply(convert)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:900\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m    898\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[1;32m    899\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchildren():\n\u001b[0;32m--> 900\u001b[0m         module\u001b[38;5;241m.\u001b[39m_apply(fn)\n\u001b[1;32m    902\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[1;32m    903\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[1;32m    904\u001b[0m         \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[1;32m    905\u001b[0m         \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    910\u001b[0m         \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[1;32m    911\u001b[0m         \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:900\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m    898\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[1;32m    899\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchildren():\n\u001b[0;32m--> 900\u001b[0m         module\u001b[38;5;241m.\u001b[39m_apply(fn)\n\u001b[1;32m    902\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[1;32m    903\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[1;32m    904\u001b[0m         \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[1;32m    905\u001b[0m         \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    910\u001b[0m         \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[1;32m    911\u001b[0m         \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
+      "    \u001b[0;31m[... skipping similar frames: Module._apply at line 900 (4 times)]\u001b[0m\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:900\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m    898\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[1;32m    899\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchildren():\n\u001b[0;32m--> 900\u001b[0m         module\u001b[38;5;241m.\u001b[39m_apply(fn)\n\u001b[1;32m    902\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[1;32m    903\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m torch\u001b[38;5;241m.\u001b[39m_has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[1;32m    904\u001b[0m         \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[1;32m    905\u001b[0m         \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    910\u001b[0m         \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[1;32m    911\u001b[0m         \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:927\u001b[0m, in \u001b[0;36mModule._apply\u001b[0;34m(self, fn, recurse)\u001b[0m\n\u001b[1;32m    923\u001b[0m \u001b[38;5;66;03m# Tensors stored in modules are graph leaves, and we don't want to\u001b[39;00m\n\u001b[1;32m    924\u001b[0m \u001b[38;5;66;03m# track autograd history of `param_applied`, so we have to use\u001b[39;00m\n\u001b[1;32m    925\u001b[0m \u001b[38;5;66;03m# `with torch.no_grad():`\u001b[39;00m\n\u001b[1;32m    926\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m--> 927\u001b[0m     param_applied \u001b[38;5;241m=\u001b[39m fn(param)\n\u001b[1;32m    928\u001b[0m p_should_use_set_data \u001b[38;5;241m=\u001b[39m compute_should_use_set_data(param, param_applied)\n\u001b[1;32m    930\u001b[0m \u001b[38;5;66;03m# subclasses may have multiple child tensors so we need to use swap_tensors\u001b[39;00m\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1326\u001b[0m, in \u001b[0;36mModule.to.<locals>.convert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m convert_to_format \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m t\u001b[38;5;241m.\u001b[39mdim() \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;241m4\u001b[39m, \u001b[38;5;241m5\u001b[39m):\n\u001b[1;32m   1320\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m t\u001b[38;5;241m.\u001b[39mto(\n\u001b[1;32m   1321\u001b[0m             device,\n\u001b[1;32m   1322\u001b[0m             dtype \u001b[38;5;28;01mif\u001b[39;00m t\u001b[38;5;241m.\u001b[39mis_floating_point() \u001b[38;5;129;01mor\u001b[39;00m t\u001b[38;5;241m.\u001b[39mis_complex() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1323\u001b[0m             non_blocking,\n\u001b[1;32m   1324\u001b[0m             memory_format\u001b[38;5;241m=\u001b[39mconvert_to_format,\n\u001b[1;32m   1325\u001b[0m         )\n\u001b[0;32m-> 1326\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m t\u001b[38;5;241m.\u001b[39mto(\n\u001b[1;32m   1327\u001b[0m         device,\n\u001b[1;32m   1328\u001b[0m         dtype \u001b[38;5;28;01mif\u001b[39;00m t\u001b[38;5;241m.\u001b[39mis_floating_point() \u001b[38;5;129;01mor\u001b[39;00m t\u001b[38;5;241m.\u001b[39mis_complex() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1329\u001b[0m         non_blocking,\n\u001b[1;32m   1330\u001b[0m     )\n\u001b[1;32m   1331\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m   1332\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(e) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot copy out of meta tensor; no data!\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
+      "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 20.00 MiB. GPU 0 has a total capacity of 6.00 GiB of which 0 bytes is free. Of the allocated memory 20.39 GiB is allocated by PyTorch, and 155.53 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)"
+     ]
+    }
+   ],
+   "source": [
+    "model = Idefics3ForConditionalGeneration.from_pretrained(source_model_id , torch_dtype=torch.bfloat16).to(DEVICE)\n",
+    "model.load_adapter(destination_model_id, device_map=\"auto\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -14400,5 +14699,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 4
 }