Upload sd_token_similarity_calculator.ipynb

sd_token_similarity_calculator.ipynb

@@ -30,7 +30,7 @@
       "source": [
         "# @title ✳️ Load/initialize values\n",
         "#Imports\n",
-        "!pip install safetensors\n",
+        "#!pip install safetensors\n",
         "from safetensors.torch import load_file\n",
         "import json , os , shelve , torch\n",
         "import pandas as pd\n",
@@ -46,7 +46,7 @@
         "    id = id-id_max\n",
         "  return id\n",
         "\n",
-        "def getPrompts(_path):\n",
+        "def getPrompts(_path, separator):\n",
         "  path = _path + '/text'\n",
         "  path_enc = _path + '/text_encodings'\n",
         "  #-----#\n",
@@ -75,6 +75,7 @@
         "      #Read the 'header' file in the JSON\n",
         "      if _index <= 0 :\n",
         "        _NUM_ITEMS = int(value)\n",
+        "        prompts[f'{index}'] = _prompts[f'{_index}'] + separator\n",
         "        index = index + 1\n",
         "        continue\n",
         "      if _index <= 1 :\n",
@@ -85,12 +86,13 @@
         "        text_encodings[f'{index-1}'] = _text_encodings[f'{_index-1}']\n",
         "        text_encodings[f'{index}'] = _text_encodings[f'{_index}']\n",
         "        #------#\n",
+        "        prompts[f'{index}'] = _prompts[f'{_index}'] + separator\n",
         "        index = index + 1\n",
         "        continue\n",
         "      #------#\n",
         "      #Read the text_encodings + prompts\n",
         "      text_encodings[f'{index}'] = _text_encodings[f'{_index}']\n",
-        "      prompts[f'{index}'] = _prompts[f'{_index}']\n",
+        "      prompts[f'{index}'] = _prompts[f'{_index}'] + separator\n",
         "      index = index + 1\n",
         "      continue\n",
         "      #-------#\n",
@@ -98,41 +100,354 @@
         "    #_text_encodings.close() #close the text_encodings file\n",
         "    file_index = file_index + 1\n",
         "  #----------#\n",
-        "  NUM_ITEMS = index\n",
+        "  NUM_ITEMS = index -1\n",
         "  return prompts , text_encodings , NUM_ITEMS\n",
-        "  #--------#\n",
+        "#--------#\n",
         "\n",
-        "#for key in prompts:\n",
-        "#  value = prompts[key]\n",
-        "#  if int(key)>=1000:break\n",
-        "#  print(value)\n",
-        "#------#\n"
+        "def append_from_url(dictA, tensA , nA , url , separator):\n",
+        "  dictB , tensB, nB = getPrompts(url, separator)\n",
+        "  dictAB = dictA\n",
+        "  tensAB = tensA\n",
+        "  nAB = nA\n",
+        "  for key in dictB:\n",
+        "    nAB = nAB + 1\n",
+        "    dictAB[f'{nA + int(key)}'] = dictB[key]\n",
+        "    tensAB[f'{nA + int(key)}'] = tensB[key]\n",
+        "  #-----#\n",
+        "  return dictAB, tensAB , nAB-1\n",
+        "#-------#\n"
       ],
       "metadata": {
         "id": "rUXQ73IbonHY"
       },
-      "execution_count": null,
+      "execution_count": 54,
       "outputs": []
     },
     {
       "cell_type": "code",
       "source": [
-        "# @title 📝 Choose which vocab to load\n",
-        "use_vocab = '🦜 fusion-t2i-prompt-features' # @param ['🦜 fusion-t2i-prompt-features']\n",
+        "# @title ✳️ Select items for the vocab\n",
+        "prompt_features = True # @param {\"type\":\"boolean\",\"placeholder\":\"🦜\"}\n",
+        "suffix = True # @param {\"type\":\"boolean\",\"placeholder\":\"🔹\"}\n",
+        "prefix = True # @param {\"type\":\"boolean\",\"placeholder\":\"🔸\"}\n",
+        "debug = False\n",
         "\n",
+        "#🔸🔹\n",
         "%cd /content/\n",
         "!git clone https://huggingface.co/datasets/codeShare/text-to-image-prompts\n",
+        "\n",
+        "#------#\n",
+        "prompts = {}\n",
+        "text_encodings = {}\n",
+        "nA = 0\n",
+        "#--------#\n",
+        "\n",
+        "if prompt_features:\n",
+        "  url = '/content/text-to-image-prompts/civitai-prompts/green'\n",
+        "  prompts , text_encodings, nA = append_from_url(prompts , text_encodings, nA , url , '')\n",
+        "  if debug:\n",
+        "    print(prompts[f'{nA}'])\n",
+        "    print(text_encodings[f'{nA}'])\n",
+        "#--------#\n",
+        "\n",
+        "if suffix :\n",
+        "  tmp = '/content/text-to-image-prompts/tokens/suffix/'\n",
+        "  for item in ['common','average','rare','weird','exotic'] :\n",
+        "    url =  tmp  + item\n",
+        "    prompts , text_encodings, nA = append_from_url(prompts , text_encodings, nA , url , '')\n",
+        "    if debug:\n",
+        "      print(prompts[f'{nA}'])\n",
+        "      print(text_encodings[f'{nA}'])\n",
+        "#------#\n",
+        "\n",
+        "if prefix :\n",
+        "  tmp = '/content/text-to-image-prompts/tokens/prefix/'\n",
+        "  for item in ['common','average','rare','weird','exotic'] :\n",
+        "    url =  tmp  + item\n",
+        "    prompts , text_encodings, nA = append_from_url(prompts , text_encodings, nA , url , '-')\n",
+        "    if debug:\n",
+        "      print(prompts[f'{nA}'])\n",
+        "      print(text_encodings[f'{nA}'])\n",
         "#------#\n",
-        "path = '/content/text-to-image-prompts/civitai-prompts/green'\n",
-        "prompts , text_encodings, NUM_VOCAB_ITEMS = getPrompts(path)\n",
         "\n",
-        "append_Whitespace = True\n"
+        "if debug:\n",
+        "  index = 0\n",
+        "  for key in prompts: index = index + 1\n",
+        "  print(index)\n",
+        "  index = 0\n",
+        "  for key in text_encodings : index = index + 1\n",
+        "  print(index)\n",
+        "#------#\n",
+        "\n",
+        "NUM_VOCAB_ITEMS = nA\n"
       ],
       "metadata": {
-        "id": "ZMG4CThUAmwW"
+        "id": "ZMG4CThUAmwW",
+        "outputId": "c34b6fcc-e4c5-4fa1-a55b-d8175ccb8f67",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
       },
-      "execution_count": null,
-      "outputs": []
+      "execution_count": 55,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "/content\n",
+            "fatal: destination path 'text-to-image-prompts' already exists and is not an empty directory.\n",
+            "reading 🦜 fusion-t2i-prompt-features-23.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-9.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-4.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-2.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-25.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-6.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-19.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-33.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-18.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-22.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-31.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-3.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-8.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-7.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-13.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-34.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-14.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-12.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-17.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-21.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-24.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-32.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-5.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-20.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-16.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-28.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-10.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-15.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-26.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-27.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-1.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-29.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-30.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🦜 fusion-t2i-prompt-features-11.json....\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text\n",
+            "/content/text-to-image-prompts/civitai-prompts/green/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-5 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-4 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-common-suffix-6 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/common/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-5 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-6 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-7 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-average-suffix-4 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/average/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-6 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-5 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-7 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-rare-suffix-4 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/rare/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-6 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-5 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-4 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-weird-suffix-7 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/weird/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-4 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-7 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-6 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔹 fusion-t2i-sd15-clip-tokens-exotic-suffix-5 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/suffix/exotic/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-common-prefix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-common-prefix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-common-prefix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/common/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-average-prefix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-average-prefix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-average-prefix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/average/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-rare-prefix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-rare-prefix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-rare-prefix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/rare/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-weird-prefix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-weird-prefix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-weird-prefix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/weird/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-exotic-prefix-1 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-exotic-prefix-2 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text_encodings\n",
+            "reading 🔸 fusion-t2i-sd15-clip-tokens-exotic-prefix-3 Tokens.json....\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text\n",
+            "/content/text-to-image-prompts/tokens/prefix/exotic/text_encodings\n"
+          ]
+        }
+      ]
     },
     {
       "cell_type": "code",
@@ -155,6 +470,7 @@
         "\n",
         "sims = torch.zeros(NUM_VOCAB_ITEMS)\n",
         "for index in range(NUM_VOCAB_ITEMS):\n",
+        "  if index<2: continue\n",
         "  text_features =  text_encodings[f'{index}']\n",
         "  sims[index] = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
         "#------#\n",
@@ -164,7 +480,7 @@
       "metadata": {
         "id": "xc-PbIYF428y"
       },
-      "execution_count": null,
+      "execution_count": 56,
       "outputs": []
     },
     {
@@ -183,7 +499,6 @@
         "# title Show the 100 most similiar suffix and prefix text-encodings to the text encoding\n",
         "RANGE = list_size\n",
         "separator = '|'\n",
-        "if append_Whitespace : separator =  ' ' + separator\n",
         "if newline_Separator : separator = separator + '\\n'\n",
         "\n",
         "_prompts = '{'\n",
@@ -213,10 +528,121 @@
         "#-------#"
       ],
       "metadata": {
-        "id": "_vnVbxcFf7WV"
+        "id": "_vnVbxcFf7WV",
+        "outputId": "b9c6a86b-1239-410b-e240-aca1c8301249",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
       },
-      "execution_count": null,
-      "outputs": []
+      "execution_count": 50,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{her car behind on the road</w>|\n",
+            "the race car driver sits behind the wheel</w>|\n",
+            "speed-|\n",
+            "driving-|\n",
+            "highway-|\n",
+            "road-|\n",
+            "fast-|\n",
+            "drive-|\n",
+            "racing car in the background</w>|\n",
+            "traffic-|\n",
+            "driver-|\n",
+            "thats-|\n",
+            "roadto-|\n",
+            "slow-|\n",
+            "cause-|\n",
+            "cruise-|\n",
+            "cyber car looking_upscenery</w>|\n",
+            "cyber car looking_upscenery</w>|\n",
+            "travel-|\n",
+            "truck-|\n",
+            "car-|\n",
+            "that-|\n",
+            "destination-|\n",
+            "way-|\n",
+            "a beautiful</w>|\n",
+            "a beautiful</w>|\n",
+            "just-|\n",
+            "grat-|\n",
+            "rolling-|\n",
+            "simply-|\n",
+            "cruis-|\n",
+            "speed</w>|\n",
+            "something-|\n",
+            "cool-|\n",
+            "roadtrip</w>|\n",
+            "onlyin-|\n",
+            "random-|\n",
+            "because-|\n",
+            "carr-|\n",
+            "shre-|\n",
+            "driving</w>|\n",
+            "funny-|\n",
+            "street-|\n",
+            "dding-|\n",
+            "aand-|\n",
+            "ontheroad</w>|\n",
+            "classic-|\n",
+            "beauti-|\n",
+            "quick-|\n",
+            "reall-|\n",
+            "wast-|\n",
+            "wait-|\n",
+            "on the crowded streets</w>|\n",
+            "haun-|\n",
+            "epic-|\n",
+            "lest-|\n",
+            "faster</w>|\n",
+            "stoo-|\n",
+            "quality-|\n",
+            "fantastic-|\n",
+            "seem-|\n",
+            "love-|\n",
+            "yeah-|\n",
+            "well-|\n",
+            "moving-|\n",
+            "nice-|\n",
+            "late-|\n",
+            "camer-|\n",
+            "sure-|\n",
+            "only-|\n",
+            "chasing-|\n",
+            "loving-|\n",
+            "the image should have</w>|\n",
+            "best-|\n",
+            "reali-|\n",
+            "takes off down the track</w>|\n",
+            "and you can't help but feel</w>|\n",
+            "dazz-|\n",
+            "followthe-|\n",
+            "awesome-|\n",
+            "words-|\n",
+            "lifein-|\n",
+            "eng-|\n",
+            "herit-|\n",
+            "oneof-|\n",
+            "but now seems to exude</w>|\n",
+            "then-|\n",
+            "beast-|\n",
+            "thisis-|\n",
+            "goodday-|\n",
+            "thisi-|\n",
+            "right-|\n",
+            "is appropriate for the time period</w>|\n",
+            "amazing-|\n",
+            "race-|\n",
+            "madd-|\n",
+            "coming-|\n",
+            "zoom-|\n",
+            "picture-|\n",
+            "ight-}\n"
+          ]
+        }
+      ]
     },
     {
       "cell_type": "markdown",
@@ -274,13 +700,13 @@
       ],
       "metadata": {
         "id": "ke6mZ1RZDOeB",
-        "outputId": "8ced884a-bf07-4fcb-c108-0f873d71a73c",
+        "outputId": "53b9280f-b8ba-4036-c720-e679bd0a162c",
         "colab": {
           "base_uri": "https://localhost:8080/",
           "height": 1000
         }
       },
-      "execution_count": 4,
+      "execution_count": 51,
       "outputs": [
         {
           "output_type": "display_data",
@@ -324,7 +750,7 @@
       "metadata": {
         "id": "rebogpoyOG8k"
       },
-      "execution_count": null,
+      "execution_count": 52,
       "outputs": []
     },
     {
@@ -343,7 +769,6 @@
         "# title Show the 100 most similiar suffix and prefix text-encodings to the text encoding\n",
         "RANGE = list_size\n",
         "separator = '|'\n",
-        "if append_Whitespace : separator =  ' ' + separator\n",
         "if newline_Separator : separator = separator + '\\n'\n",
         "\n",
         "_prompts = '{'\n",
@@ -377,114 +802,114 @@
         "colab": {
           "base_uri": "https://localhost:8080/"
         },
-        "outputId": "37351bed-c5e2-4554-c5e0-a9dc84da700b"
+        "outputId": "68e866ad-9cc7-43f6-ad67-d72c28fa0b46"
       },
-      "execution_count": 6,
+      "execution_count": 53,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "{beautiful avatar pictures |\n",
-            "purple hair crowned standing in storm background |\n",
-            "beautiful celebrity futuristic sci-fi |\n",
-            "by magali villeneuve |\n",
-            "visually striking spectacle inspired by the works |\n",
-            "visually striking spectacle inspired by the works |\n",
-            "a beautiful  female warrior |\n",
-            "a sexy scifi warrior |\n",
-            "a sexy scifi warrior |\n",
-            "film still from halo live action adaptation |\n",
-            "cinematic film still from captain marvel |\n",
-            "beautiful female warrior |\n",
-            "beautiful  female warrior |\n",
-            "film still from halo live-action movie adaptation |\n",
-            "outlandish costume design |\n",
-            "beautiful indian warrior queen |\n",
-            "of female space soldier |\n",
-            "blue light on her face she appears calm |\n",
-            "a female scifi warrior |\n",
-            "a female scifi warrior |\n",
-            "nebula in her streak hair |\n",
-            "of brown skinned indian warrior queen |\n",
-            "played by young dove cameron |\n",
-            "has runes on her body |\n",
-            "has runes on her body |\n",
-            "beautiful light makeup female sorceress |\n",
-            "a gorgeous female void thrall |\n",
-            "a gorgeous female void thrall |\n",
-            "beautiful female elf queen |\n",
-            "captivating mystique |\n",
-            "captivating mystique |\n",
-            "symbolizing her role as the goddess |\n",
-            "character integrated into the background |\n",
-            "swirling black light around the character |\n",
-            "very beautiful jean grey wearing |\n",
-            "lightly blued metal armor |\n",
-            "multiple different characters in the background |\n",
-            "cinematic still from conan |\n",
-            "yo person as dark elf queen |\n",
-            "trending at cgsociety |\n",
-            "femaleastronaut exalted human futuristic warrior |\n",
-            "femaleastronaut exalted human futuristic warrior |\n",
-            "epic fantasy greek priestess |\n",
-            "epic fantasy greek priestess |\n",
-            "female draenei world |\n",
-            "genetically engineered soldiers |\n",
-            "genetically engineered soldiers |\n",
-            "visually striking scene the lighting |\n",
-            "the female soldier marches in formation |\n",
-            "the female soldier marches in formation |\n",
-            "revealing costume design |\n",
-            "pandora_smith_magister |\n",
-            "pandora_smith_magister |\n",
-            "his sorceress in the back ground |\n",
-            "gorgeous muscular elven ukrainian |\n",
-            "gorgeous muscular elven ukrainian |\n",
-            "water elemental officer jenny |\n",
-            "norse female goddess |\n",
-            "matte fantasy painting |\n",
-            "periwinkle purple skin |\n",
-            "of norse female goddess |\n",
-            "mujer de ojos rojos y pelo azulado |\n",
-            "strength the battle scene around her |\n",
-            "a beautiful young redhead warrior |\n",
-            "a beautiful young redhead warrior |\n",
-            "moody cinematic epic concept art |\n",
-            "hypnotically beautiful wood elf in |\n",
-            "hypnotically beautiful wood elf in |\n",
-            "loraemmawatsonlora_v |\n",
-            "alphonse mucha cinematic epic + rule |\n",
-            "alphonse mucha cinematic epic + rule |\n",
-            "an actress standing behind |\n",
-            "fking_scifi_v amazing |\n",
-            "shine like sapphires |\n",
-            "female elemental water wizard |\n",
-            "female elemental water wizard |\n",
-            "beautiful character design |\n",
-            "female warriors protecting an underwater temple |\n",
-            "indian mary_winstead |\n",
-            "sarah kerrigan queen |\n",
-            "sarah kerrigan queen |\n",
-            "the female barbarian stands tall |\n",
-            "widowmaker from overwatchscarlett johannson |\n",
-            "intricate costume design |\n",
-            "award winning character concept art of |\n",
-            "movie still from braveheart |\n",
-            "movie still from braveheart |\n",
-            "an extremely beautiful young female elf |\n",
-            "an extremely beautiful young female elf |\n",
-            "cinematic character design d |\n",
-            "dark purple bodypaint |\n",
-            "dark purple bodypaint |\n",
-            "a daring reimagining |\n",
-            "captivating film still |\n",
-            "glowing dark blue skin |\n",
-            "of sexy cyberpunk female mage |\n",
-            "military men gaze at her longingly |\n",
-            "military men gaze at her longingly |\n",
-            "female space soldier |\n",
-            "norse goddess fighting in}\n"
+            "{beautiful avatar pictures</w>|\n",
+            "purple hair crowned standing in storm background</w>|\n",
+            "warcraft</w>|\n",
+            "beautiful celebrity futuristic sci-fi</w>|\n",
+            "by magali villeneuve</w>|\n",
+            "visually striking spectacle inspired by the works</w>|\n",
+            "visually striking spectacle inspired by the works</w>|\n",
+            "a beautiful  female warrior</w>|\n",
+            "a sexy scifi warrior</w>|\n",
+            "a sexy scifi warrior</w>|\n",
+            "film still from halo live action adaptation</w>|\n",
+            "cinematic film still from captain marvel</w>|\n",
+            "beautiful female warrior</w>|\n",
+            "beautiful  female warrior</w>|\n",
+            "film still from halo live-action movie adaptation</w>|\n",
+            "outlandish costume design</w>|\n",
+            "defend-|\n",
+            "beautiful indian warrior queen</w>|\n",
+            "genie</w>|\n",
+            "of female space soldier</w>|\n",
+            "vfx</w>|\n",
+            "blue light on her face she appears calm</w>|\n",
+            "a female scifi warrior</w>|\n",
+            "a female scifi warrior</w>|\n",
+            "orian</w>|\n",
+            "nebula in her streak hair</w>|\n",
+            "wall-|\n",
+            "of brown skinned indian warrior queen</w>|\n",
+            "played by young dove cameron</w>|\n",
+            "thora</w>|\n",
+            "has runes on her body</w>|\n",
+            "has runes on her body</w>|\n",
+            "beautiful light makeup female sorceress</w>|\n",
+            "a gorgeous female void thrall</w>|\n",
+            "a gorgeous female void thrall</w>|\n",
+            "beautiful female elf queen</w>|\n",
+            "captivating mystique</w>|\n",
+            "captivating mystique</w>|\n",
+            "symbolizing her role as the goddess</w>|\n",
+            "blu-|\n",
+            "character integrated into the background</w>|\n",
+            "swirling black light around the character</w>|\n",
+            "very beautiful jean grey wearing</w>|\n",
+            "lightly blued metal armor</w>|\n",
+            "multiple different characters in the background</w>|\n",
+            "raid</w>|\n",
+            "tributes</w>|\n",
+            "cinematic still from conan</w>|\n",
+            "yo person as dark elf queen</w>|\n",
+            "descendants</w>|\n",
+            "trending at cgsociety</w>|\n",
+            "exodus</w>|\n",
+            "scarlett</w>|\n",
+            "femaleastronaut exalted human futuristic warrior</w>|\n",
+            "femaleastronaut exalted human futuristic warrior</w>|\n",
+            "indigo</w>|\n",
+            "epic fantasy greek priestess</w>|\n",
+            "epic fantasy greek priestess</w>|\n",
+            "female draenei world</w>|\n",
+            "genetically engineered soldiers</w>|\n",
+            "genetically engineered soldiers</w>|\n",
+            "visually striking scene the lighting</w>|\n",
+            "the female soldier marches in formation</w>|\n",
+            "the female soldier marches in formation</w>|\n",
+            "revealing costume design</w>|\n",
+            "pandora_smith_magister</w>|\n",
+            "pandora_smith_magister</w>|\n",
+            "his sorceress in the back ground</w>|\n",
+            "tali</w>|\n",
+            "gorgeous muscular elven ukrainian</w>|\n",
+            "gorgeous muscular elven ukrainian</w>|\n",
+            "womenshistorymonth</w>|\n",
+            "water elemental officer jenny</w>|\n",
+            "ontari-|\n",
+            "bleedblue</w>|\n",
+            "norse female goddess</w>|\n",
+            "matte fantasy painting</w>|\n",
+            "thanos</w>|\n",
+            "periwinkle purple skin</w>|\n",
+            "xmen</w>|\n",
+            "of norse female goddess</w>|\n",
+            "mujer de ojos rojos y pelo azulado</w>|\n",
+            "strength the battle scene around her</w>|\n",
+            "a beautiful young redhead warrior</w>|\n",
+            "a beautiful young redhead warrior</w>|\n",
+            "moody cinematic epic concept art</w>|\n",
+            "hypnotically beautiful wood elf in</w>|\n",
+            "hypnotically beautiful wood elf in</w>|\n",
+            "loraemmawatsonlora_v</w>|\n",
+            "alphonse mucha cinematic epic + rule</w>|\n",
+            "alphonse mucha cinematic epic + rule</w>|\n",
+            "an actress standing behind</w>|\n",
+            "fking_scifi_v amazing</w>|\n",
+            "shine like sapphires</w>|\n",
+            "defence</w>|\n",
+            "female elemental water wizard</w>|\n",
+            "female elemental water wizard</w>|\n",
+            "beautiful character design</w>|\n",
+            "horde</w>|\n",
+            "female warriors protecting an underwater temple</w>}\n"
           ]
         }
       ]