Annotate more images and retest

assets/images/vegan_ingredients/annotations.json

@@ -3,9 +3,9 @@
         {
             "file_name": "fridge-splendid.jpg",
             "ingredients": [
-                "oatmilk",
+                "oat milk",
                 "brown rice taco bowl",
-                "brocolli",
+                "broccoli",
                 "asparagus",
                 "avocado",
                 "nut butter"
@@ -26,7 +26,7 @@
                 "soy milk",
                 "bananas",
                 "tomatoes",
-                "raisin bran",
+                "cereal",
                 "cucumber",
                 "carrot"
             ]
@@ -35,7 +35,7 @@
             "file_name": "veggie-fridge.jpg",
             "ingredients": [
                 "avocado",
-                "oatmilk",
+                "oat milk",
                 "cashew yogurt",
                 "ginger",
                 "lettuce"
@@ -62,6 +62,90 @@
                 "broccoli",
                 "kale"
             ]
+        },
+        {
+            "file_name": "pantry/1.jpg",
+            "ingredients": [
+                "salsa",
+                "coconut milk",
+                "tomato soup",
+                "tomato paste",
+                "cashews"
+            ]
+        },
+        {
+            "file_name": "pantry/2.jpg",
+            "ingredients": [
+                "peanut butter",
+                "nuts"
+            ]
+        },
+        {
+            "file_name": "pantry/3.jpg",
+            "ingredients": [
+                "egg replacer"
+            ]
+        },
+        {
+            "file_name": "pantry/4.jpg",
+            "ingredients": [
+                "peanut butter",
+                "coconut milk",
+                "potatoes",
+                "sweet potatoes",
+                "muffins",
+                "chex cereal",
+                "crackers",
+                "salsa"
+            ]
+        },
+        {
+            "file_name": "pantry/5.jpg",
+            "ingredients": [
+                "pasta",
+                "coffee",
+                "flax seeds"
+            ]
+        },
+        {
+            "file_name": "pantry/6.jpg",
+            "ingredients": [
+                "apple cider vinegar",
+                "pasta  sauce",
+                "maple syrup",
+                "peppermint tea"
+            ]
+        },
+        {
+            "file_name": "pantry/7.jpg",
+            "ingredients": [
+                "almonds",
+                "wheat gluten",
+                "penne pasta",
+                "bulgur",
+                "yellow rice",
+                "pasta",
+                "walnuts",
+                "grains"
+            ]
+        },
+        {
+            "file_name": "pantry/8.jpg",
+            "ingredients": [
+                "rice",
+                "beans",
+                "blackeye peas"
+            ]
+        },
+        {
+            "file_name": "pantry/9.jpg",
+            "ingredients": [
+                "soy sauce",
+                "lemonade",
+                "blackeye peas",
+                "rice",
+                "beans"
+            ]
         }
     ]
 }

nbs/04_gptv.ipynb

@@ -39,6 +39,7 @@
     "import json\n",
     "import logging\n",
     "import os\n",
+    "import statistics\n",
     "import time\n",
     "from glob import glob\n",
     "from logging.handlers import RotatingFileHandler\n",
@@ -58,9 +59,9 @@
    "outputs": [],
    "source": [
     "#| hide\n",
+    "import markdown\n",
     "from dotenv import load_dotenv\n",
-    "from ipywidgets import widgets\n",
-    "import markdown"
+    "from ipywidgets import widgets"
    ]
   },
   {
@@ -584,6 +585,7 @@
     "# function takes a prompt, tries it on all annotated images, evaluates keywords presence (lowered), ouputs data for analytics\n",
     "def eval_ingredient_prompt(prompt: str):\n",
     "    data = []\n",
+    "    annotations = load_json(f\"{SAMPLE_IMG_DIR}/annotations.json\")\n",
     "    for i, entry in enumerate(annotations[\"images\"]):\n",
     "        logging.info(f\"On image {i} of {len(annotations['images'])}\")\n",
     "        msg = get_gptv_completion(prompt, f\"{SAMPLE_IMG_DIR}/{entry['file_name']}\")\n",
@@ -598,9 +600,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "prompt1_data = eval_ingredient_prompt(\n",
-    "    \"What vegan ingredients are in the image? Ouput as JSON list.\"\n",
-    ")"
+    "prompt1 = \"What vegan ingredients are in the image? Ouput as JSON list.\"\n",
+    "prompt1_data = eval_ingredient_prompt(prompt1)"
    ]
   },
   {
@@ -638,7 +639,35 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "932277cffe1543eda51131cbf28d84a8",
+       "model_id": "8a46c8962bae4467a0c5808f693a4aaf",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HTML(value='<h3>Prompt: What vegan ingredients are in the image? Ouput as JSON list.</h3>')"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "91867d0c4167416f85ac077c33888a3b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HTML(value='<h3>Mean ingredients found: 58%</h3>')"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f85834f8d12e4f9e8e73cc496bf41e12",
        "version_major": 2,
        "version_minor": 0
       },
@@ -674,6 +703,8 @@
     "\n",
     "def display_ingredient_results(metrics_data):\n",
     "    vboxes = []\n",
+    "    mean_ing_covered_p = round(statistics.mean([d['percent_ing_covered'] for d in metrics_data]))\n",
+    "    display(widgets.HTML(value=f\"<h3>Mean ingredients found: {mean_ing_covered_p}%</h3>\"))\n",
     "    for d in metrics_data:\n",
     "        img = widgets.Image(\n",
     "            value=open(f\"{SAMPLE_IMG_DIR}/{d['img']}\", \"rb\").read(),\n",
@@ -692,7 +723,7 @@
     "        w_percent_ing = widgets.HTML(\n",
     "            value=f\"<p style='font-weight: bold;'>Ingredients found: {d['percent_ing_covered']}%</p>\"\n",
     "        )\n",
-    "        w_completion = widgets.HTML(value=markdown.markdown(d['gptv_completion']))\n",
+    "        w_completion = widgets.HTML(value=markdown.markdown(d[\"gptv_completion\"]))\n",
     "        vbox = widgets.VBox(\n",
     "            [img, w_percent_ing] + texts + [w_completion],\n",
     "            layout=widgets.Layout(\n",
@@ -704,7 +735,7 @@
     "    hbox = widgets.HBox(vboxes, layout=widgets.Layout(overflow=\"scroll\"))\n",
     "    display(hbox)\n",
     "\n",
-    "\n",
+    "display(widgets.HTML(value=f\"<h3>Prompt: {prompt1}</h3>\"))\n",
     "display_ingredient_results(analyze_ingredient_results(prompt1_data))"
    ]
   },
@@ -718,6 +749,18 @@
     "```\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Normalize .jpg file ending\n",
+    "```sh\n",
+    "for file in *.JPG; do\n",
+    "    mv \"$file\" \"${file%.JPG}.jpg\"\n",
+    "done\n",
+    "```"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,