Update the vectorstore creation logic -- create a lighter version of chromadb for production code

prototypes/build_rag_datastore.ipynb

@@ -10,19 +10,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 7,
    "id": "b4e0a36f-f8c2-42d1-8920-54396e5737c5",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Volumes/VTG/Dev/C_5/Projects/week8/.venv/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import os\n",
     "from tqdm import tqdm\n",
@@ -37,7 +28,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 8,
    "id": "66a1a814-76ef-44d0-b22f-b3568b7195fc",
    "metadata": {},
    "outputs": [],
@@ -53,7 +44,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 9,
    "id": "fd156583-053e-402d-88a8-aac74e703894",
    "metadata": {},
    "outputs": [
@@ -82,18 +73,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 10,
+   "id": "bb0bd7c2-2194-482a-8afb-c1cb4643875b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Want to create cvector db for local development or production?\n",
+    "### If DEV_MODE = false(for production code), lighter chromadb will be created due to the hosting file size limit\n",
+    "DEV_MODE = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
    "id": "35c3e944-1c63-4c65-b32b-99d8b807d671",
    "metadata": {},
    "outputs": [],
    "source": [
     "with open(\"train.pkl\", \"rb\") as f:\n",
-    "    train = pickle.load(f)"
+    "    train = pickle.load(f) if DEV_MODE else pickle.load(f)[:100000]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 12,
    "id": "e89e5566-0974-4524-a2cc-ec45a64bcbe6",
    "metadata": {},
    "outputs": [
@@ -116,6 +119,27 @@
     "print(train[0].prompt)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "038b88c2-ced5-43e0-9f21-735e4460e7db",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "100000"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(train)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "c75e1399-a9b1-466e-9a80-2f61d947defc",
@@ -128,14 +152,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "5820500d-f2e3-43ad-b0b2-85d73d49112f",
    "metadata": {},
    "outputs": [],
    "source": [
-    "DEV_MODE = False\n",
+    "\n",
     "DB_PATH = DB if DEV_MODE else PRODUCTION_DB\n",
-    "collection_name = \"products\" if DEV_MODE else \"products-deployment\"\n",
+    "collection_name = \"products\"\n",
     "\n",
     "client = chromadb.PersistentClient(path=DB_PATH)\n",
     "collection = client.get_or_create_collection(collection_name)    "
@@ -143,30 +167,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "be6d4079-a9e1-442b-9825-2e8f6ffc09c8",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Creating new collection.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "### Use the existing collection, otherwise create it if none of it.\n",
-    "collection_name = \"products\"\n",
+    "###### NOTE: simply get_or_create_collection does the job perfectly without this messy lines of code\n",
+    "\n",
+    "# collection_name = \"products\"\n",
     "\n",
-    "existing_collection_names = client.list_collections()\n",
+    "# existing_collection_names = client.list_collections()\n",
     "\n",
-    "if any(c.name == collection_name for c in existing_collection_names): \n",
-    "    print(\"Collection already exists. Using it.\")\n",
-    "    collection = client.get_collection(collection_name)\n",
-    "else: \n",
-    "    print(\"Creating new collection.\")\n",
-    "    collection = client.create_collection(collection_name)"
+    "# if any(c.name == collection_name for c in existing_collection_names): \n",
+    "#     print(\"Collection already exists. Using it.\")\n",
+    "#     collection = client.get_collection(collection_name)\n",
+    "# else: \n",
+    "#     print(\"Creating new collection.\")\n",
+    "#     collection = client.create_collection(collection_name)"
    ]
   },
   {
@@ -179,7 +197,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "0328e557-fbbb-4412-9c18-dcab38975839",
    "metadata": {},
    "outputs": [],
@@ -203,7 +221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "d5366cea-3468-4443-a2c6-dce30f774c17",
    "metadata": {},
    "outputs": [],
@@ -222,23 +240,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "5c5c230b-f58a-4617-b0b3-3f928c7b96b1",
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'model' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[12]\u001b[39m\u001b[32m, line 1\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m1\u001b[39m \u001b[43mhow_similar\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mJava\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mC++\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m      2\u001b[39m how_similar(\u001b[33m\"\u001b[39m\u001b[33mJava\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mmug\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m      3\u001b[39m how_similar(\u001b[33m\"\u001b[39m\u001b[33mCup of Java\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mmug\u001b[39m\u001b[33m\"\u001b[39m)\n",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[11]\u001b[39m\u001b[32m, line 8\u001b[39m, in \u001b[36mhow_similar\u001b[39m\u001b[34m(text1, text2)\u001b[39m\n\u001b[32m      7\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mhow_similar\u001b[39m(text1, text2):\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m     vector1, vector2 = \u001b[43mmodel\u001b[49m.encode([text1, text2])\n\u001b[32m      9\u001b[39m     similarity = cosine_similarity(vector1, vector2)\n\u001b[32m     10\u001b[39m     \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mSimilarity between \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtext1\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m and \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtext2\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msimilarity*\u001b[32m100\u001b[39m\u001b[38;5;132;01m:\u001b[39;00m\u001b[33m.1f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m%\u001b[39m\u001b[33m\"\u001b[39m)\n",
-      "\u001b[31mNameError\u001b[39m: name 'model' is not defined"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "how_similar(\"Java\", \"C++\")\n",
     "how_similar(\"Java\", \"mug\")\n",
@@ -247,7 +252,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "id": "b0b743ff-2606-47fc-be86-69c62ec788b1",
    "metadata": {},
    "outputs": [],
@@ -268,18 +273,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "373ba2ae-db0c-4ff0-a27f-9a442a6590a0",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|████████████████████████████████████████████████████████████████| 150/150 [17:43<00:00,  7.09s/it]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "NUMBER_OF_DOCS = len(train)\n",
     "\n",