added model.safetensors

.gitattributes

@@ -1 +1,2 @@
 *.png filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,3 @@
-model.safetensors
 
 # python and vs code stuff
 __pycache__/
@@ -13,6 +12,5 @@ __pycache__/
 # venv
 venv/
 
-*.safetensors
 
 log/

moondream2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd6a554b4453fb3853c8d9a2ee562bfe4c565013d8bca1153f5e9f4c40d3b4f5
+size 3854534792

notes.ipynb

@@ -72,23 +72,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "RuntimeError",
-     "evalue": "Error(s) in loading state_dict for MoondreamModel:\n\tWhile copying the parameter named \"region.size_features\", whose dimensions in the model are torch.Size([2, 256]) and whose dimensions in the checkpoint are torch.Size([2, 256]), an exception occurred : ('CUDA error: out of memory\\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\\n',).",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mRuntimeError\u001b[39m                              Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 14\u001b[39m\n\u001b[32m     12\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01msafetensors\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mtorch\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m load_model\n\u001b[32m     13\u001b[39m weights_path = \u001b[33m\"\u001b[39m\u001b[33mmoondream2/model2.safetensors\u001b[39m\u001b[33m\"\u001b[39m  \u001b[38;5;66;03m# Path to your local weights file\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m14\u001b[39m state_dict = \u001b[43mload_model\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweights_path\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     15\u001b[39m new_state_dict = {}\n\u001b[32m     16\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m state_dict.items():\n\u001b[32m     17\u001b[39m     \u001b[38;5;66;03m# Remove 'model.' prefix if it exists\u001b[39;00m\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/safetensors/torch.py:205\u001b[39m, in \u001b[36mload_model\u001b[39m\u001b[34m(model, filename, strict, device)\u001b[39m\n\u001b[32m    203\u001b[39m model_state_dict = model.state_dict()\n\u001b[32m    204\u001b[39m to_removes = _remove_duplicate_names(model_state_dict, preferred_names=state_dict.keys())\n\u001b[32m--> \u001b[39m\u001b[32m205\u001b[39m missing, unexpected = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mload_state_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate_dict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m    206\u001b[39m missing = \u001b[38;5;28mset\u001b[39m(missing)\n\u001b[32m    207\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m to_remove_group \u001b[38;5;129;01min\u001b[39;00m to_removes.values():\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/torch/nn/modules/module.py:2593\u001b[39m, in \u001b[36mModule.load_state_dict\u001b[39m\u001b[34m(self, state_dict, strict, assign)\u001b[39m\n\u001b[32m   2585\u001b[39m         error_msgs.insert(\n\u001b[32m   2586\u001b[39m             \u001b[32m0\u001b[39m,\n\u001b[32m   2587\u001b[39m             \u001b[33m\"\u001b[39m\u001b[33mMissing key(s) in state_dict: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m.format(\n\u001b[32m   2588\u001b[39m                 \u001b[33m\"\u001b[39m\u001b[33m, \u001b[39m\u001b[33m\"\u001b[39m.join(\u001b[33mf\u001b[39m\u001b[33m'\u001b[39m\u001b[33m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mk\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\u001b[33m'\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m missing_keys)\n\u001b[32m   2589\u001b[39m             ),\n\u001b[32m   2590\u001b[39m         )\n\u001b[32m   2592\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(error_msgs) > \u001b[32m0\u001b[39m:\n\u001b[32m-> \u001b[39m\u001b[32m2593\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m   2594\u001b[39m         \u001b[33m\"\u001b[39m\u001b[33mError(s) in loading state_dict for \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[33m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[33m\"\u001b[39m.format(\n\u001b[32m   2595\u001b[39m             \u001b[38;5;28mself\u001b[39m.\u001b[34m__class__\u001b[39m.\u001b[34m__name__\u001b[39m, \u001b[33m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\t\u001b[39;00m\u001b[33m\"\u001b[39m.join(error_msgs)\n\u001b[32m   2596\u001b[39m         )\n\u001b[32m   2597\u001b[39m     )\n\u001b[32m   2598\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m _IncompatibleKeys(missing_keys, unexpected_keys)\n",
-      "\u001b[31mRuntimeError\u001b[39m: Error(s) in loading state_dict for MoondreamModel:\n\tWhile copying the parameter named \"region.size_features\", whose dimensions in the model are torch.Size([2, 256]) and whose dimensions in the checkpoint are torch.Size([2, 256]), an exception occurred : ('CUDA error: out of memory\\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1\\nCompile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.\\n',)."
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
@@ -100,29 +86,37 @@
     "\n",
     "config = MoondreamConfig()\n",
     "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "model = MoondreamModel(config, setup_caches=False).to(device)\n",
+    "model = MoondreamModel(config, setup_caches=False)\n",
     "from safetensors.torch import load_model\n",
     "weights_path = \"moondream2/model2.safetensors\"  # Path to your local weights file\n",
     "state_dict = load_model(model, weights_path)\n",
-    "new_state_dict = {}\n",
-    "for key, value in state_dict.items():\n",
-    "    # Remove 'model.' prefix if it exists\n",
-    "    if key.startswith('model.'):\n",
-    "        new_key = key[6:]  # Skip the first 6 characters ('model.')\n",
-    "    else:\n",
-    "        new_key = key\n",
-    "    new_state_dict[new_key] = value\n",
-    "state_dict = new_state_dict\n",
-    "missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=True)\n",
     "model._setup_caches()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "ename": "OutOfMemoryError",
+     "evalue": "CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 3.69 GiB of which 129.50 MiB is free. Including non-PyTorch memory, this process has 3.56 GiB memory in use. Of the allocated memory 3.47 GiB is allocated by PyTorch, and 13.58 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mOutOfMemoryError\u001b[39m                          Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/torch/nn/modules/module.py:1355\u001b[39m, in \u001b[36mModule.to\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m   1352\u001b[39m         \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m   1353\u001b[39m             \u001b[38;5;28;01mraise\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1355\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mconvert\u001b[49m\u001b[43m)\u001b[49m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/torch/nn/modules/module.py:915\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m    913\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m recurse:\n\u001b[32m    914\u001b[39m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.children():\n\u001b[32m--> \u001b[39m\u001b[32m915\u001b[39m         \u001b[43mmodule\u001b[49m\u001b[43m.\u001b[49m\u001b[43m_apply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfn\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    917\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcompute_should_use_set_data\u001b[39m(tensor, tensor_applied):\n\u001b[32m    918\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m torch._has_compatible_shallow_copy_type(tensor, tensor_applied):\n\u001b[32m    919\u001b[39m         \u001b[38;5;66;03m# If the new tensor has compatible tensor type as the existing tensor,\u001b[39;00m\n\u001b[32m    920\u001b[39m         \u001b[38;5;66;03m# the current behavior is to change the tensor in-place using `.data =`,\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m    925\u001b[39m         \u001b[38;5;66;03m# global flag to let the user control whether they want the future\u001b[39;00m\n\u001b[32m    926\u001b[39m         \u001b[38;5;66;03m# behavior of overwriting the existing tensor or not.\u001b[39;00m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/torch/nn/modules/module.py:942\u001b[39m, in \u001b[36mModule._apply\u001b[39m\u001b[34m(self, fn, recurse)\u001b[39m\n\u001b[32m    938\u001b[39m \u001b[38;5;66;03m# Tensors stored in modules are graph leaves, and we don't want to\u001b[39;00m\n\u001b[32m    939\u001b[39m \u001b[38;5;66;03m# track autograd history of `param_applied`, so we have to use\u001b[39;00m\n\u001b[32m    940\u001b[39m \u001b[38;5;66;03m# `with torch.no_grad():`\u001b[39;00m\n\u001b[32m    941\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.no_grad():\n\u001b[32m--> \u001b[39m\u001b[32m942\u001b[39m     param_applied = \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparam\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    943\u001b[39m p_should_use_set_data = compute_should_use_set_data(param, param_applied)\n\u001b[32m    945\u001b[39m \u001b[38;5;66;03m# subclasses may have multiple child tensors so we need to use swap_tensors\u001b[39;00m\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/Desktop/moondream/venv/lib/python3.13/site-packages/torch/nn/modules/module.py:1341\u001b[39m, in \u001b[36mModule.to.<locals>.convert\u001b[39m\u001b[34m(t)\u001b[39m\n\u001b[32m   1334\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m convert_to_format \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m t.dim() \u001b[38;5;129;01min\u001b[39;00m (\u001b[32m4\u001b[39m, \u001b[32m5\u001b[39m):\n\u001b[32m   1335\u001b[39m         \u001b[38;5;28;01mreturn\u001b[39;00m t.to(\n\u001b[32m   1336\u001b[39m             device,\n\u001b[32m   1337\u001b[39m             dtype \u001b[38;5;28;01mif\u001b[39;00m t.is_floating_point() \u001b[38;5;129;01mor\u001b[39;00m t.is_complex() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m   1338\u001b[39m             non_blocking,\n\u001b[32m   1339\u001b[39m             memory_format=convert_to_format,\n\u001b[32m   1340\u001b[39m         )\n\u001b[32m-> \u001b[39m\u001b[32m1341\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   1342\u001b[39m \u001b[43m        \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1343\u001b[39m \u001b[43m        \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mis_floating_point\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mis_complex\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m   1344\u001b[39m \u001b[43m        \u001b[49m\u001b[43mnon_blocking\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   1345\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m   1346\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m   1347\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(e) == \u001b[33m\"\u001b[39m\u001b[33mCannot copy out of meta tensor; no data!\u001b[39m\u001b[33m\"\u001b[39m:\n",
+      "\u001b[31mOutOfMemoryError\u001b[39m: CUDA out of memory. Tried to allocate 200.00 MiB. GPU 0 has a total capacity of 3.69 GiB of which 129.50 MiB is free. Including non-PyTorch memory, this process has 3.56 GiB memory in use. Of the allocated memory 3.47 GiB is allocated by PyTorch, and 13.58 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)"
+     ]
+    }
+   ],
+   "source": [
+    "model.to(device)"
+   ]
   },
   {
    "cell_type": "code",
@@ -409,7 +403,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "venv12",
+   "display_name": "venv",
    "language": "python",
    "name": "python3"
   },
@@ -423,7 +417,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.10"
+   "version": "3.13.3"
   }
  },
  "nbformat": 4,