Upload sd_token_similarity_calculator.ipynb

sd_token_similarity_calculator.ipynb

@@ -25,6 +25,15 @@
         "id": "L7JTcbOdBPfh"
       }
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "THIS IS AN OLD VERSION. UP TO DATE VERSION CAN BE FOUND HERE : https://huggingface.co/datasets/codeShare/text-to-image-prompts/tree/main/Google%20Colab%20Notebooks"
+      ],
+      "metadata": {
+        "id": "W0T0dlDuCPTN"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
@@ -122,10 +131,31 @@
         "!git clone https://huggingface.co/datasets/codeShare/text-to-image-prompts\n"
       ],
       "metadata": {
-        "id": "rUXQ73IbonHY"
+        "id": "rUXQ73IbonHY",
+        "outputId": "5ba68c03-2c3e-4f47-c307-712ff431f7ec",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
       },
-      "execution_count": null,
-      "outputs": []
+      "execution_count": 1,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "/content\n",
+            "Cloning into 'text-to-image-prompts'...\n",
+            "remote: Enumerating objects: 2372, done.\u001b[K\n",
+            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
+            "remote: Total 2372 (delta 0), reused 0 (delta 0), pack-reused 2368 (from 1)\u001b[K\n",
+            "Receiving objects: 100% (2372/2372), 20.75 MiB | 12.08 MiB/s, done.\n",
+            "Resolving deltas: 100% (417/417), done.\n",
+            "Updating files: 100% (1301/1301), done.\n",
+            "Filtering content: 100% (578/578), 2.21 GiB | 36.41 MiB/s, done.\n"
+          ]
+        }
+      ]
     },
     {
       "cell_type": "code",
@@ -238,10 +268,98 @@
         "NUM_VOCAB_ITEMS = nA\n"
       ],
       "metadata": {
-        "id": "ZMG4CThUAmwW"
+        "id": "ZMG4CThUAmwW",
+        "outputId": "40d41f92-d461-4517-caa9-94951abccdca",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 1000
+        }
       },
-      "execution_count": null,
-      "outputs": []
+      "execution_count": 2,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "reading 🎀 fusion-t2i-danbooru-tags-1.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-5.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-9.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-18.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-17.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-13.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-2.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-12.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-6.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-11.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-4.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-3.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-15.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-8.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-10.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-20.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-19.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-21.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-16.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-14.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n",
+            "reading 🎀 fusion-t2i-danbooru-tags-7.json....\n",
+            "/content/text-to-image-prompts/danbooru/text\n",
+            "/content/text-to-image-prompts/danbooru/text_encodings\n"
+          ]
+        },
+        {
+          "output_type": "error",
+          "ename": "FileNotFoundError",
+          "evalue": "[Errno 2] No such file or directory: '/content/text-to-image-prompts/tokens/suffix/common/text'",
+          "traceback": [
+            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+            "\u001b[0;32m<ipython-input-2-490cc35e84d5>\u001b[0m in \u001b[0;36m<cell line: 83>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     85\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'common'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'average'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rare'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'weird'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'exotic'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0mtmp\u001b[0m  \u001b[0;34m+\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m     \u001b[0mprompts\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtext_encodings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mappend_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompts\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtext_encodings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m \u001b[0;31m#------#\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;32m<ipython-input-1-8be73a8413a2>\u001b[0m in \u001b[0;36mappend_from_url\u001b[0;34m(dictA, tensA, nA, url, separator)\u001b[0m\n\u001b[1;32m     76\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mappend_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdictA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mnA\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mseparator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m   \u001b[0mdictB\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mtensB\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetPrompts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseparator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m   \u001b[0mdictAB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdictA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m   \u001b[0mtensAB\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtensA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;32m<ipython-input-1-8be73a8413a2>\u001b[0m in \u001b[0;36mgetPrompts\u001b[0;34m(_path, separator)\u001b[0m\n\u001b[1;32m     27\u001b[0m   \u001b[0m_text_encodings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m   \u001b[0;31m#-----#\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mfilename\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'{path}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'reading {filename}....'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/text-to-image-prompts/tokens/suffix/common/text'"
+          ]
+        }
+      ]
     },
     {
       "cell_type": "code",