Upload Nitral_Token_Surgeon_Example.ipynb

TokenSurgeon-Example/Nitral_Token_Surgeon_Example.ipynb

@@ -0,0 +1,182 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "L4",
+      "machine_shape": "hm"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Download / install required dependencies."
+      ],
+      "metadata": {
+        "id": "4nHRbUX16c0C"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Clone and install\n",
+        "!git clone https://github.com/cg123/mergekit.git\n",
+        "!cd mergekit && pip install -q -e .\n",
+        "!cd ..\n",
+        "\n",
+        "# Remove git-related files to clean up\n",
+        "!rm -rf mergekit/.git mergekit/.gitignore mergekit/.gitattributes\n",
+        "\n",
+        "# Optional: clean README, .md files, tests, etc., if you want minimal bloat\n",
+        "!rm -rf mergekit/tests mergekit/*.md\n",
+        "\n",
+        "# Install other dependencies\n",
+        "!pip install huggingface_hub hf_xet"
+      ],
+      "metadata": {
+        "id": "XQm5_Xtz09yE"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Download HF Models to ./cache/"
+      ],
+      "metadata": {
+        "id": "a3HqMjNQ6a01"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WTa6H9ij06FX"
+      },
+      "outputs": [],
+      "source": [
+        "from huggingface_hub import snapshot_download\n",
+        "\n",
+        "model_names = [\n",
+        "    \"Nitral-AI/Irixxed-Magcap-12B-0.1a\",\n",
+        "    \"Entropicengine/Pinecone-Rune-12b\"\n",
+        "]\n",
+        "\n",
+        "for model_name in model_names:\n",
+        "    # Set cache_dir to avoid loading into memory\n",
+        "    cache_dir = f\"./cache/{model_name.replace('/', '_')}\"\n",
+        "\n",
+        "    # Download the entire repository using snapshot_download\n",
+        "    snapshot_download(repo_id=model_name, local_dir=cache_dir, local_dir_use_symlinks=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        " # Clean pip & HF junk so system disk doesn't run out on t4.\n"
+      ],
+      "metadata": {
+        "id": "KHcohLhKVMfn"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip cache purge\n",
+        "!rm -rf /root/.cache/huggingface\n",
+        "!rm -rf ~/.cache/pip\n",
+        "!rm -rf ~/.cache/torch_extensions\n",
+        "!rm -rf ~/.nv"
+      ],
+      "metadata": {
+        "id": "dICO9UxoVJ1l",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "19877723-ce37-49d5-a8c2-b81a258de863"
+      },
+      "execution_count": 3,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Files removed: 90\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Start Merge-kit's Token-Surgeon with --cosine-similarity -v -k 64 --cuda --low-cpu-memory so it works on a t4.\n",
+        "./cache/ for model download location\n",
+        "\n",
+        "Base model, donor model. [Donor embeddings are mixed using a linear relationship based on similarity and then applied to base]"
+      ],
+      "metadata": {
+        "id": "H6gBOEjA6VhN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "!mergekit-tokensurgeon ./cache/Entropicengine_Pinecone-Rune-12b ./cache/Nitral-AI_Irixxed-Magcap-12B-0.1a ./postop -v -k 64 --cosine-similarity --cuda --low-cpu-memory\n"
+      ],
+      "metadata": {
+        "id": "vfjXBAGV1qVr"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Upload to Huggingface"
+      ],
+      "metadata": {
+        "id": "JfYTwAzr6PR8"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from google.colab import userdata\n",
+        "from huggingface_hub import HfApi, HfFolder\n",
+        "\n",
+        "# Fetch token from Colab secrets\n",
+        "HF_TOKEN = userdata.get('HF_TOKEN')  # Now it's assigned\n",
+        "\n",
+        "# Initialize API with token\n",
+        "api = HfApi(token=HF_TOKEN)\n",
+        "\n",
+        "repo_name = \"Nitral-AI/Pinecone-Rune-12b-Token-Surgery-Chatml\"\n",
+        "\n",
+        "# Create the repo if it doesn't already exist\n",
+        "api.create_repo(repo_id=repo_name, private=True, exist_ok=True)\n",
+        "\n",
+        "# Upload the local folder contents to the repo\n",
+        "api.upload_folder(\n",
+        "    folder_path=\"./postop\",\n",
+        "    repo_id=repo_name,\n",
+        ")"
+      ],
+      "metadata": {
+        "id": "zKoL6Ouf4pLI"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}