Upload sd_token_similarity_calculator.ipynb

sd_token_similarity_calculator.ipynb

@@ -46,8 +46,18 @@
         "NUM_PREFIX = 13662\n",
         "NUM_SUFFIX = 32901\n",
         "\n",
-        "PREFIX_ENC_VOCAB = 'encoded_prefix_to_girl'\n",
-        "SUFFIX_ENC_VOCAB = 'encoded_suffix'\n",
+        "PREFIX_ENC_VOCAB = ['encoded_prefix_to_girl',]\n",
+        "SUFFIX_ENC_VOCAB = [\n",
+        "    'from_-encoded_suffix',\n",
+        "    'a_-_encoded_suffix' ,\n",
+        "    'by_-encoded_suffix' ,\n",
+        "    'encoded_suffix-_like']\n",
+        "\n",
+        "# Make sure these match above results\n",
+        "NUM_PREFIX_LISTS = 1\n",
+        "NUM_SUFFIX_LISTS = 4\n",
+        "#-----#\n",
+        "\n",
         "\n",
         "#Import the vocab.json\n",
         "import json\n",
@@ -134,33 +144,21 @@
         "  return ' ' #<---- return whitespace if out of bounds\n",
         "#--------#\n",
         "\n",
+        "\n",
+        "def _modulus(_id,id_max):\n",
+        "  id = _id\n",
+        "  while(id>id_max):\n",
+        "    id = id-id_max\n",
+        "  return id\n",
+        "\n",
         "#print(get_token(35894))\n"
       ],
       "metadata": {
         "id": "Ch9puvwKH1s3",
-        "collapsed": true,
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "outputId": "2333a33b-1344-4a14-bee6-060d98167715"
+        "collapsed": true
       },
-      "execution_count": 1,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Cloning into 'sd_tokens'...\n",
-            "remote: Enumerating objects: 72, done.\u001b[K\n",
-            "remote: Counting objects: 100% (69/69), done.\u001b[K\n",
-            "remote: Compressing objects: 100% (69/69), done.\u001b[K\n",
-            "remote: Total 72 (delta 24), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
-            "Unpacking objects: 100% (72/72), 1.34 MiB | 1.65 MiB/s, done.\n",
-            "Filtering content: 100% (10/10), 899.76 MiB | 50.12 MiB/s, done.\n",
-            "/content/sd_tokens\n"
-          ]
-        }
-      ]
+      "execution_count": null,
+      "outputs": []
     },
     {
       "cell_type": "code",
@@ -316,18 +314,125 @@
       "outputs": []
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
       "source": [
-        "Below image interrogator appends CLIP tokens to either end of the 'must_contain' text , and seeks to maximize similarity with the image encoding.\n",
+        "# @title 📝 Prompt similarity:  Order pre-made text_encodings\n",
+        "prompt = \"photo of a banana\" # @param {\"type\":\"string\",\"placeholder\":\"Write a prompt\"}\n",
+        "from transformers import AutoTokenizer\n",
+        "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
+        "from transformers import  CLIPProcessor, CLIPModel\n",
+        "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\" , clean_up_tokenization_spaces = True)\n",
+        "model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
         "\n",
-        "It takes a long while to check all the tokens (too long!) so this cell only samples a range of the 49K available tokens.\n",
+        "# Get text features for user input\n",
+        "inputs = tokenizer(text = prompt, padding=True, return_tensors=\"pt\")\n",
+        "text_features_A = model.get_text_features(**inputs)\n",
+        "text_features_A = text_features_A/text_features_A.norm(p=2, dim=-1, keepdim=True)\n",
+        "name_A = prompt\n",
+        "#------#\n",
         "\n",
-        "You can run this cell, then paste the result into the 'must_contain' box , and then run the cell again.\n",
+        "# Load the .db file for prefix encodings\n",
+        "import shelve\n",
+        "_iters = -1\n",
+        "RANGE = NUM_PREFIX\n",
+        "NUM_PREFIX_LISTS = 1\n",
+        "dots = results_sim = torch.zeros(RANGE*NUM_PREFIX_LISTS)\n",
+        "for _PREFIX_ENC_VOCAB in PREFIX_ENC_VOCAB:\n",
+        "  _iters = _iters + 1\n",
+        "  d = shelve.open(_PREFIX_ENC_VOCAB)\n",
+        "  for _index in range(RANGE):\n",
+        "    index = _iters*RANGE + _index\n",
+        "    text_features = d[f'{_index}']\n",
+        "    text_features = text_features/text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    sim = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
+        "    dots[index] = sim\n",
+        "  #----#\n",
+        "  d.close() #close the file\n",
+        "#------#\n",
+        "prefix_sorted, prefix_indices = torch.sort(dots,dim=0 , descending=True)\n",
+        "#------#\n",
         "\n",
-        "Check the sd_tokens folder for stored .db files from running the '⚡ Get similiar tokens' cell. These can be used in the ⚡+🖼️ -> 📝 Token-Sampling Image interrogator cell\n"
+        "# Load the .db file for prefix encodings\n",
+        "import shelve\n",
+        "_iters = -1\n",
+        "RANGE = NUM_SUFFIX\n",
+        "dots = results_sim = torch.zeros(RANGE*NUM_SUFFIX_LISTS)\n",
+        "for _SUFFIX_ENC_VOCAB in SUFFIX_ENC_VOCAB:\n",
+        "  _iters = _iters + 1\n",
+        "  d = shelve.open(_SUFFIX_ENC_VOCAB)\n",
+        "  for _index in range(RANGE):\n",
+        "    index = _iters*RANGE + _index\n",
+        "    text_features = d[f'{_index}']\n",
+        "    text_features = text_features/text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    sim = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
+        "    dots[index] = sim\n",
+        "  #----#\n",
+        "  d.close() #close the file\n",
+        "#------#\n",
+        "suffix_sorted, suffix_indices = torch.sort(dots,dim=0 , descending=True)\n",
+        "#------#\n",
+        "\n",
+        "#Print the results\n",
+        "#'from_-encoded_suffix',\n",
+        "#'a_-_encoded_suffix' ,\n",
+        "#'by_-encoded_suffix' ,\n",
+        "#'encoded_suffix-_like'\n",
+        "\n",
+        "# title Show the 100 most similiar suffix and prefix text-encodings to the text encoding\n",
+        "RANGE = 100\n",
+        "_suffixes = '{'\n",
+        "_sims =  '{'\n",
+        "for index in range(RANGE):\n",
+        "  id = int(suffix_indices[index])\n",
+        "  ahead = \"from \"\n",
+        "  behind = \"\"\n",
+        "  if(id>NUM_SUFFIX*1):\n",
+        "    ahead = \"a \"\n",
+        "  if(id>NUM_SUFFIX*2):\n",
+        "    ahead = \"by \"\n",
+        "  if(id>NUM_SUFFIX*3):\n",
+        "    ahead = \"\"\n",
+        "    behind = \"like\"\n",
+        "  id = _modulus(id,NUM_SUFFIX)\n",
+        "  #------#\n",
+        "  sim = suffix_sorted[index].item()\n",
+        "  name = ahead + get_suffix(id) + behind\n",
+        "  if(get_suffix(id) == ' '): name = ahead + f'{id}' + behind\n",
+        "  _suffixes = _suffixes + name + '|'\n",
+        "  _sims = _sims + f'{round(sim*100,2)} %' + '|'\n",
+        "#------#\n",
+        "_suffixes = (_suffixes + '}').replace('|}', '}')\n",
+        "_sims = (_sims + '}').replace('|}', '}')\n",
+        "\n",
+        "print('most similiar suffix items to prompt : ' + _suffixes)\n",
+        "print('similarity % for suffix items : ' + _sims)\n",
+        "print('')\n",
+        "\n",
+        "#-------#\n",
+        "\n",
+        "_prefixes = '{'\n",
+        "for index in range(RANGE):\n",
+        "  id = f'{prefix_indices[index]}'\n",
+        "  #sim = prefix_sorted[index]\n",
+        "  name = get_prefix(id)\n",
+        "  _prefixes = _prefixes + name + '|'\n",
+        "#------#\n",
+        "_prefixes = (_prefixes + '}').replace('|}', '}')\n",
+        "print('most similiar prefix suffix to image : ' + _prefixes)\n"
+      ],
+      "metadata": {
+        "id": "xc-PbIYF428y"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Below are the Image interrogators"
       ],
       "metadata": {
-        "id": "IUCuV9RtQpBn"
+        "id": "qZvLkJCtGC89"
       }
     },
     {
@@ -397,7 +502,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# @title Order pre-made text_encodings to image similarity\n",
+        "# @title 🖼️ Image similarity : Order pre-made text_encodings\n",
         "from transformers import AutoTokenizer\n",
         "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
         "from transformers import  CLIPProcessor, CLIPModel\n",
@@ -447,7 +552,7 @@
     {
       "cell_type": "code",
       "source": [
-        "# @title Show the 10 most similiar suffix and prefix text-encodings to the image encoding\n",
+        "# @title 🖼️ Show the 10 most similiar suffix and prefix text-encodings to the image encoding\n",
         "\n",
         "_suffixes = '{'\n",
         "for index in range(20):\n",
@@ -895,7 +1000,7 @@
       "metadata": {
         "id": "9ZiTsF9jV0TV"
       },
-      "execution_count": 6,
+      "execution_count": null,
       "outputs": []
     },
     {