Migrated RAG to its package

.gitignore

@@ -3,4 +3,5 @@ __pycache__/
 .venv/ 
 .env
 .chainlit/
-.files/
+.files/
+.vscode/

notebooks/transcript_rag.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 121,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 122,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 123,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -43,7 +43,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 124,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -59,7 +59,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 125,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -87,10 +87,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 126,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
+    "from ast import Dict\n",
     "import json\n",
     "filename = \"../data/test.json\"\n",
     "\n",
@@ -99,17 +100,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 127,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain_experimental.text_splitter import SemanticChunker\n",
-    "from langchain_openai.embeddings import OpenAIEmbeddings\n",
-    "from pstuts_rag.datastore import transcripts_load\n",
-    "\n",
-    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n",
-    "docs_chunks_semantic = transcripts_load(data,embeddings)"
-   ]
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_values"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
   },
   {
    "cell_type": "markdown",
@@ -127,26 +132,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 128,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_qdrant import QdrantVectorStore\n",
-    "from qdrant_client import QdrantClient\n",
-    "from qdrant_client.http.models import Distance, VectorParams\n",
-    "\n",
-    "qdrantclient = QdrantClient(\":memory:\")\n",
+    "from pstuts_rag.rag import RetrieverFactory\n",
     "\n",
-    "vectorstore = pstuts_rag.datastore.initialize_vectorstore(\n",
-    "    client=qdrantclient,\n",
-    "    collection_name=f\"{params.filename}_qdrant\",\n",
-    "    embeddings=embeddings,\n",
-    ")\n",
-    "\n",
-    "_ = vectorstore.add_documents(documents=docs_chunks_semantic)\n",
-    "retriever =vectorstore.as_retriever(\n",
-    "    search_kwargs={\"k\": params.n_context_docs}\n",
-    ")"
+    "retriever_factory = RetrieverFactory()\n",
+    "retriever_factory.add_docs(raw_docs=data)"
    ]
   },
   {
@@ -159,301 +152,111 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 139,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "from langchain.prompts import ChatPromptTemplate\n",
-    "\n",
-    "prompt_template = ChatPromptTemplate.from_messages([\n",
-    "    (\"system\", \"\"\"\\\n",
-    "You are a helpful and friendly Photoshop expert.\n",
-    "\n",
-    "Your job is to answer user questions based **only** on transcript excerpts from training videos. These transcripts include **timestamps** that indicate when in the video the information was spoken.\n",
-    "\n",
-    "The transcript is from **spoken audio**, so it may include informal phrasing, filler words, or fragmented sentences. You may interpret meaning **only to the extent it is clearly implied**, but you must not add new information or invent details.\n",
-    "\n",
-    "✅ Your Responsibilities\n",
-    "\n",
-    "1. Use **only** the transcript to answer.\n",
-    "2. If a clear answer is **not** present in the transcript, respond exactly:  \n",
-    "   \"I don't know. This isn’t covered in the training videos.\"\n",
-    "3. When appropriate, include the **timestamp** of relevant information in your answer to help the user locate it in the original video.\n",
-    "4. Do **not** make assumptions or draw on outside knowledge.\n",
-    "\n",
-    "💡 Style & Formatting Tips\n",
-    "\n",
-    "- Use a step-by-step format when explaining procedures 📋.\n",
-    "- Add relevant emojis for clarity and friendliness 🎨🖱️🔧.\n",
-    "- Keep your answers short, clear, and conversational.\n",
-    "- The input timestamps will be in seconds. When reporting timestamps, convert them into minute:seconds format.\n",
-    "\n",
-    "⛔ Never Do This\n",
-    "\n",
-    "- ❌ Don't guess or summarize from general knowledge.\n",
-    "- ❌ Don’t fabricate steps, names, or features not in the transcript.\n",
-    "- ❌ Don’t omit the fallback response when required.\n",
-    "\"\"\"),\n",
-    "    (\"user\",\"\"\"\\\n",
-    "\n",
-    "### Question\n",
-    "{question}\n",
-    "\n",
-    "NEVER invent the explanation. ALWAYS use ONLY the context information.\n",
-    "\n",
-    "### Context\n",
-    "{context}\n",
+    "## Generation\n",
     "\n",
-    "\"\"\")])\n"
+    "We will use a 4.1-nano to generate answers."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 140,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def compile_references(context):\n",
-    "    references = [ \n",
-    "                    {k: doc.metadata[k] for k in (\"title\",\"source\",\"start\",\"stop\")} \n",
-    "                    for doc in context\n",
-    "    ] \n",
-    "    print(type(references))\n",
-    "    return json.dumps(references,indent=2)\n"
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(model=\"gpt-4.1-mini\",temperature=0)"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": 26,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "## Generation\n",
+    "from pstuts_rag.rag import RAGChainFactory\n",
     "\n",
-    "We will use a 4.1-nano to generate answers."
+    "rag_factory = RAGChainFactory(retriever=retriever_factory.get_retriever())\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 141,
+   "execution_count": 49,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_openai import ChatOpenAI\n",
     "\n",
-    "llm = ChatOpenAI(model=\"gpt-4.1-mini\",temperature=0)"
+    "get_videos = rag_factory.get_rag_chain(llm)\n",
+    "    \n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 142,
+   "execution_count": 50,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'list'>\n",
-      "('Layers are the building blocks of any image in Photoshop CC. You can think '\n",
-      " 'of layers like separate flat panes of glass stacked on top of each other. '\n",
-      " 'Each layer contains separate pieces of content. Some parts of a layer can be '\n",
-      " 'transparent, allowing you to see through to the layers below. This setup '\n",
-      " 'lets you edit parts of an image independently without affecting the rest of '\n",
-      " 'the image. You manage and work with layers in the Layers panel, where you '\n",
-      " 'can toggle their visibility on and off using the Eye icon. (See explanation '\n",
-      " 'around 0:28 to 1:00 and 1:25 to 2:32) 🎨🖼️\\n'\n",
-      " 'References:\\n'\n",
-      " '[\\n'\n",
-      " '  {\\n'\n",
-      " '    \"title\": \"Understand layers\",\\n'\n",
-      " '    \"source\": '\n",
-      " '\"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\\n'\n",
-      " '    \"start\": 0.47,\\n'\n",
-      " '    \"stop\": 62.14\\n'\n",
-      " '  },\\n'\n",
-      " '  {\\n'\n",
-      " '    \"title\": \"Understand layers\",\\n'\n",
-      " '    \"source\": '\n",
-      " '\"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\\n'\n",
-      " '    \"start\": 85.75,\\n'\n",
-      " '    \"stop\": 152.97\\n'\n",
-      " '  }\\n'\n",
-      " ']')\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "from operator import itemgetter\n",
-    "from langchain.schema.output_parser import StrOutputParser\n",
-    "from langchain_core.runnables import RunnableLambda\n",
-    "\n",
-    "form_context = RunnableLambda(itemgetter(\"question\")) | {\n",
-    "    \"context\":  retriever, \n",
-    "    \"question\": RunnablePassthrough() \n",
-    "    } \n",
-    "\n",
-    "answer_chain = prompt_template | llm | StrOutputParser()\n",
-    "\n",
-    "get_videos =   form_context | \\\n",
-    "    {\"input\":RunnablePassthrough(),\"answer\": answer_chain} |\\\n",
-    "    RunnableLambda( lambda d: \n",
-    "        {**d[\"input\"], \"answer\": d[\"answer\"] + \n",
-    "         \"\\nReferences:\\n\" +\n",
-    "         compile_references(d[\"input\"][\"context\"]) \n",
-    "        } )\n",
-    "    \n",
-    "\n",
-    "\n",
-    "val = get_videos.invoke({\"question\":\"What are layers\"})\n",
-    "pp(val[\"answer\"])"
+    "val = get_videos.invoke({\"question\":\"What are layers\"})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 143,
+   "execution_count": 52,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "('Layers are the building blocks of any image in Photoshop CC. You can think '\n",
-      " 'of layers like separate flat panes of glass stacked on top of each other. '\n",
-      " 'Each layer contains separate pieces of content. Some parts of a layer can be '\n",
-      " 'transparent, allowing you to see through to the layers below. This setup '\n",
-      " 'lets you edit parts of an image independently without affecting the rest of '\n",
-      " 'the image. You manage and work with layers in the Layers panel, where you '\n",
-      " 'can toggle their visibility on and off using the Eye icon. (See explanation '\n",
-      " 'around 0:28 to 1:00 and 1:25 to 2:32) 🎨🖼️\\n'\n",
-      " 'References:\\n'\n",
-      " '[\\n'\n",
-      " '  {\\n'\n",
-      " '    \"title\": \"Understand layers\",\\n'\n",
-      " '    \"source\": '\n",
-      " '\"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\\n'\n",
-      " '    \"start\": 0.47,\\n'\n",
-      " '    \"stop\": 62.14\\n'\n",
-      " '  },\\n'\n",
-      " '  {\\n'\n",
-      " '    \"title\": \"Understand layers\",\\n'\n",
-      " '    \"source\": '\n",
-      " '\"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\\n'\n",
-      " '    \"start\": 85.75,\\n'\n",
-      " '    \"stop\": 152.97\\n'\n",
-      " '  }\\n'\n",
-      " ']')\n"
-     ]
+     "data": {
+      "text/plain": [
+       "{'refusal': None,\n",
+       " 'context': [Document(metadata={'video_id': 19172, 'title': 'Understand layers', 'desc': 'Learn what layers are and why they are so useful.', 'length': '00:04:44.75', 'source': 'https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4', 'speech_start_stop_times': [[0.47, 3.41], [3.81, 9.13], [9.309999, 15.01], [15.299999, 20.57], [20.88, 23.3], [23.83, 27.93], [29.38, 32.79], [32.96, 33.92], [34.43, 40.21], [41.91, 45.37], [45.88, 49.01], [49.54, 55.130001], [55.72, 58.49], [58.72, 62.14]], 'start': 0.47, 'stop': 62.14, '_id': '9124467014f5417eb5f90a7f350a7f4e', '_collection_name': '2f1798fb-3bc9-4ece-9bc5-1f465fc4b821_qdrant'}, page_content=\"Layers are the building blocks of any image in Photoshop CC. So, it's important to understand, what layers are and why to use them - which we'll cover in this video. If you're following along, open this layered image from the downloadable practice files for this tutorial. You might think of layers like separate flat pints of glass, stacked one on top of the other. Each layer contains separate pieces of content. To get a sense of how layers are constructed, let's take a look at this Layers panel. I've closed my other panels, so that we can focus on the Layers panel. But you can skip that. By the way: If your Layers panel isn't showing, go up to the Window menu and choose Layers from there. The Layers panel is where you go to select and work with layers. In this image there are 4 layers, each with separate content. If you click the Eye icon to the left of a layer, you can toggle the visibility of that layer off and on. So, I'm going to turn off the visibility of the tailor layer. And keep your eye on the image, so you can see what's on that layer.\"),\n",
+       "  Document(metadata={'video_id': 19172, 'title': 'Understand layers', 'desc': 'Learn what layers are and why they are so useful.', 'length': '00:04:44.75', 'source': 'https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4', 'speech_start_stop_times': [[85.75, 88.659999], [89.42, 100.11], [101.469999, 108.64], [109.09, 117.459999], [117.75, 129.45], [129.97, 133.37], [133.73, 143.98], [144.76, 152.97]], 'start': 85.75, 'stop': 152.97, '_id': '2b41e3b5ed894393a4d50b55820888c9', '_collection_name': '2f1798fb-3bc9-4ece-9bc5-1f465fc4b821_qdrant'}, page_content=\"Now let's take a look at just one layer, the tailor layer. A quick way to turn off all the layers except the tailor layer, is to hold down the Option key on the Mac, or the ALT key on the PC, and click on the Eye icon to the left of the tailor layer. In the Document window, you can see that this layer contains just the one small photo surrounded by a gray and white checkerboard pattern. That pattern represents transparent pixels, which allow us to see down through the corresponding part of this layer to the content of the layers below. So, let's turn that content back on by going back to the Layers panel, again holding the Option key on the Mac or the ALT key on the PC and clicking on the Eye icon to the left of the tailor layer. And all the other layers and their Eye icons come back into view. So again: You might think of layers like a stack of pints of glass, each with its own artwork and in some cases transparent areas that let you see down through to the layers below. The biggest benefit of having items on separate layers like this, is that you'll be able to edit pieces of an image independently without affecting the rest of the image.\")],\n",
+       " 'question': 'What are layers'}"
+      ]
+     },
+     "execution_count": 52,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
-    "pp(val[\"answer\"])"
+    "val.additional_kwargs"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 144,
+   "execution_count": 53,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "('Layers are the building blocks of any image in Photoshop CC. 🖼️ They can be '\n",
-      " 'thought of as separate flat pints of glass, stacked one on top of the other. '\n",
-      " 'Each layer contains separate pieces of content, and some layers may have '\n",
-      " 'transparent areas that let you see through to the layers below. The Layers '\n",
-      " 'panel is where you select and work with layers, and you can toggle their '\n",
-      " 'visibility by clicking the Eye icon. The main benefit of layers is that they '\n",
-      " 'allow you to edit parts of an image independently without affecting the '\n",
-      " 'rest. \\n'\n",
-      " '\\n'\n",
-      " '📺 Watch the full explanation in the video titled \"Understand layers\" here: '\n",
-      " '[https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4](https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4)')\n"
-     ]
-    }
-   ],
-   "source": [
-    "pp(value.content)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 145,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'generate' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[145]\u001b[39m\u001b[32m, line 13\u001b[39m\n\u001b[32m     10\u001b[39m     context: List[Document]\n\u001b[32m     11\u001b[39m     response: \u001b[38;5;28mstr\u001b[39m\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m graph_builder = StateGraph(State).add_sequence([retrieve, \u001b[43mgenerate\u001b[49m ])\n\u001b[32m     14\u001b[39m graph_builder.add_edge(START, \u001b[33m\"\u001b[39m\u001b[33mretrieve\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m     15\u001b[39m graph = graph_builder.compile()\n",
-      "\u001b[31mNameError\u001b[39m: name 'generate' is not defined"
+      "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
+      "\n",
+      "Layers are the building blocks of any image in Photoshop CC. You can think of layers like separate flat panes of glass stacked one on top of the other. Each layer contains separate pieces of content. Some parts of a layer can be transparent, allowing you to see through to the layers below. This setup lets you edit parts of an image independently without affecting the rest of the image. You work with layers in the Layers panel, where you can toggle their visibility on and off to see what each layer contains (like turning off the tailor layer to see what’s on it). The transparent areas are shown as a gray and white checkerboard pattern. This concept is explained around 0:47 to 3:41 and 85:75 to 152:97 in the video. 🎨🖼️\n",
+      "**References**:\n",
+      "[\n",
+      "  {\n",
+      "    \"title\": \"Understand layers\",\n",
+      "    \"source\": \"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\n",
+      "    \"start\": 0.47,\n",
+      "    \"stop\": 62.14\n",
+      "  },\n",
+      "  {\n",
+      "    \"title\": \"Understand layers\",\n",
+      "    \"source\": \"https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4\",\n",
+      "    \"start\": 85.75,\n",
+      "    \"stop\": 152.97\n",
+      "  }\n",
+      "]\n"
      ]
     }
    ],
    "source": [
-    "from langgraph.graph import START, StateGraph\n",
-    "from typing_extensions import List, TypedDict,Annotated\n",
-    "from langchain_core.documents import Document\n",
-    "from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n",
-    "from langchain_openai.chat_models import ChatOpenAI\n",
-    "import operator\n",
-    "\n",
-    "class State(TypedDict):\n",
-    "    question: str\n",
-    "    context: List[Document]\n",
-    "    response: str\n",
-    "        \n",
-    "graph_builder = StateGraph(State).add_sequence([retrieve, generate ])\n",
-    "graph_builder.add_edge(START, \"retrieve\")\n",
-    "graph = graph_builder.compile()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 53,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.schema.output_parser import StrOutputParser\n",
-    "response = graph.invoke({\"question\" : \"What is the layer in Photoshop\"})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "response.keys()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "type(response)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pp(response)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "response.keys()"
+    "val.pretty_print()"
    ]
   },
   {

pstuts_rag/pstuts_rag/prompt_templates.py

@@ -0,0 +1,53 @@
+from typing import List, Tuple
+
+RAG_PROMPT_TEMPLATES: List[Tuple[str, str]] = []
+
+RAG_PROMPT_TEMPLATES.append(
+    (
+        "system",
+        """\
+You are a helpful and friendly Photoshop expert.
+
+Your job is to answer user questions based **only** on transcript excerpts from training videos. These transcripts include **timestamps** that indicate when in the video the information was spoken.
+
+The transcript is from **spoken audio**, so it may include informal phrasing, filler words, or fragmented sentences. You may interpret meaning **only to the extent it is clearly implied**, but you must not add new information or invent details.
+
+✅ Your Responsibilities
+
+1. Use **only** the transcript to answer.
+2. If a clear answer is **not** present in the transcript, respond exactly:  
+   "I don't know. This isn’t covered in the training videos."
+3. When appropriate, include the **timestamp** of relevant information in your answer to help the user locate it in the original video.
+4. Do **not** make assumptions or draw on outside knowledge.
+
+💡 Style & Formatting Tips
+
+- Use a step-by-step format when explaining procedures 📋.
+- Add relevant emojis for clarity and friendliness 🎨🖱️🔧.
+- Keep your answers short, clear, and conversational.
+- The input timestamps will be in seconds. When reporting timestamps, convert them into minute:seconds format.
+
+⛔ Never Do This
+
+- ❌ Don't guess or summarize from general knowledge.
+- ❌ Don’t fabricate steps, names, or features not in the transcript.
+- ❌ Don’t omit the fallback response when required.
+""",
+    )
+)
+
+RAG_PROMPT_TEMPLATES.append(
+    (
+        "user",
+        """\
+### Question
+{question}
+
+NEVER invent the explanation. ALWAYS use ONLY the context information.
+
+### Context
+{context}
+
+""",
+    )
+)

pstuts_rag/pstuts_rag/rag.py

@@ -1,67 +1,160 @@
+import json
+import uuid
+from operator import itemgetter
 from typing import List, Optional, Union
-from langchain_experimental.text_splitter import SemanticChunker
-from langchain_openai.embeddings import OpenAIEmbeddings
-
-from langchain_qdrant import QdrantVectorStore
-from qdrant_client import QdrantClient
-from qdrant_client.http.models import Distance, VectorParams
-
-from .datastore import transcripts_load, initialize_vectorstore
 
-from langchain_experimental.text_splitter import SemanticChunker
-from langchain_openai.embeddings import OpenAIEmbeddings
+from langchain.schema.output_parser import StrOutputParser
 from langchain_core.documents import Document
+from langchain_core.runnables import (
+    Runnable,
+    RunnableLambda,
+    RunnablePassthrough,
+)
 
-from pstuts_rag.loader import VideoTranscriptBulkLoader, VideoTranscriptLoader
-
+from langchain_openai.embeddings import OpenAIEmbeddings
 from langchain_qdrant import QdrantVectorStore
 from qdrant_client import QdrantClient
-from qdrant_client.http.models import Distance, VectorParams
 
-from langchain_core.retrievers import VectorStoreRetriever
+from langchain.prompts import ChatPromptTemplate
+from langchain_core.vectorstores import VectorStoreRetriever
+from langchain_openai import ChatOpenAI
 
+from .datastore import initialize_vectorstore, transcripts_load
+from .prompt_templates import RAG_PROMPT_TEMPLATES
 
-import uuid
+from langchain_core.language_models.base import BaseLanguageModel
+from langchain_core.messages import AIMessage
 
 
-class RAGFactory:
+class RetrieverFactory:
 
     embeddings: OpenAIEmbeddings
     docs: List[Document]
-    qdrantclient: QdrantClient
+    qdrant_client: QdrantClient
     name: str
-    vectorstore: QdrantVectorStore
-    
+    vector_store: QdrantVectorStore
 
     def __init__(
         self,
-        raw_docs: List[Documents],        
-        embeddings: OpenAIEmbeddings=OpenAIEmbeddings(model="text-embedding-3-small"),
-        qdrantclient: QdrantClient = QdrantClient(":memory:"),
-        name: str = uuid.uuid4(),
+        embeddings: OpenAIEmbeddings = OpenAIEmbeddings(
+            model="text-embedding-3-small"
+        ),
+        qdrant_client: QdrantClient = QdrantClient(location=":memory:"),
+        name: str = str(object=uuid.uuid4()),
     ) -> None:
 
         self.embeddings = embeddings
         self.name = name
-        self.qdrantclient = qdrantclient
-        self.vectorstore = initialize_vectorstore(
-            client=self.qdrantclient,
+        self.qdrant_client = qdrant_client
+        self.vector_store = initialize_vectorstore(
+            client=self.qdrant_client,
             collection_name=f"{self.name}_qdrant",
             embeddings=self.embeddings,
         )
         self.docs = []
 
+    def add_docs(self, raw_docs) -> None:
 
-    def add_docs(raw_docs: List[Documents]):
-        
-        docs = transcripts_load(raw_docs, self.embeddings)
-        self.docs.append( docs )
-        _ = self.vectorstore.add_documents(documents=docs)
-        
-    def clear() -> bool:
+        docs: List[Document] = transcripts_load(
+            json_transcripts=raw_docs, embeddings=self.embeddings
+        )
+        self.docs.extend(docs)
+        _ = self.vector_store.add_documents(documents=docs)
+
+    def clear(self) -> bool:
         self.docs = []
-        return self.vectorstore.delete()
-                
-        
-    def get_retriever(self, n_context_docs=2) -> VectorStoreRetriever
-        return self.vectorstore.as_retriever(search_kwargs={"k": n_context_docs})
+        return True if self.vector_store.delete() else False
+
+    def get_retriever(self, n_context_docs: int = 2) -> VectorStoreRetriever:
+        return self.vector_store.as_retriever(
+            search_kwargs={"k": n_context_docs}
+        )
+
+
+class RAGChainFactory:
+
+    format_query = RunnableLambda(itemgetter("question"))
+    retriever: VectorStoreRetriever
+    add_context_to_query: Runnable
+    prompt_template: Runnable
+    answer_chain: Runnable
+    llm: ChatOpenAI
+
+    rag_chain: Runnable
+
+    @staticmethod
+    def compile_references(context: List[Document]) -> str:
+        references = [
+            {k: doc.metadata[k] for k in ("title", "source", "start", "stop")}
+            for doc in context
+        ]
+        return json.dumps(references, indent=2)
+
+    pack_references: RunnableLambda = RunnableLambda(
+        lambda d: {
+            **d["input"],
+            "answer": d["answer"]
+            + "\nReferences:\n"
+            + RAGChainFactory.compile_references(
+                context=d["input"]["context"]
+            ),
+        }
+    )
+
+    @staticmethod
+    def pack_references2(msg_dict: Dict[str, Any]) -> AIMessage:
+
+        answer: AIMessage = msg_dict["answer"]
+        input = msg_dict["input"]
+
+        references = RAGChainFactory.compile_references(
+            context=input["context"]
+        )
+
+        text_w_references = f"{answer.content}\n**References**:\n{references}"
+
+        output: AIMessage = answer.model_copy(
+            update={
+                "content": text_w_references,
+                "additional_kwargs": {
+                    **answer.additional_kwargs,
+                    "context": input["context"],
+                    "question": input["question"],
+                },
+            }
+        )
+
+        return output
+
+    def __init__(
+        self,
+        retriever: VectorStoreRetriever,
+    ) -> None:
+
+        self.retriever = retriever
+
+        self.prepare_query = {
+            "context": retriever,
+            "question": RunnablePassthrough(),
+        }
+
+        self.prompt_template = ChatPromptTemplate.from_messages(
+            RAG_PROMPT_TEMPLATES
+        )
+
+    def get_rag_chain(
+        self,
+        llm: BaseLanguageModel = ChatOpenAI(
+            model="gpt-4.1-mini", temperature=0
+        ),
+    ) -> Runnable:
+
+        self.answer_chain = self.prompt_template | llm
+        self.rag_chain = (
+            self.format_query
+            | self.prepare_query
+            | {"input": RunnablePassthrough(), "answer": self.answer_chain}
+            | self.pack_references2
+        )
+
+        return self.rag_chain

pyproject.toml

@@ -33,6 +33,9 @@ dependencies = [
     "unstructured>=0.17.2",
     "uvicorn>=0.25.0,<0.26.0",
     "websockets==14.2",
+    "langchain-core>=0.3.59",
+    "isort>=6.0.1",
+    "pylint-venv>=3.0.4",
 ]
 authors = [{ name = "Marko Budisic", email = "mbudisic@gmail.com" }]
 license = "MIT"
@@ -44,44 +47,44 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = ["pstuts_rag/pstuts_rag"]
 
-# [project.optional-dependencies]
-# dev = [
-#     "pytest>=7.0.0",
-#     "black>=22.0.0",
-#     "flake8>=4.0.0",
-#     "mypy>=0.900",
-# ]
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "black>=22.0.0",
+    "flake8>=4.0.0",
+    "mypy>=0.900",
+]
 
-# [tool.ruff]
-# line-length = 88
-# target-version = "py313"
-# select = ["E", "F", "I", "N", "W"]
-# ignore = []
+[tool.ruff]
+line-length = 79
+target-version = "py313"
+select = ["E", "F", "I", "N", "W"]
+ignore = []
 
-# [tool.ruff.isort]
-# known-first-party = ["src"]
+[tool.ruff.isort]
+known-first-party = ["src"]
 
-# [tool.black]
-# line-length = 88
-# target-version = ["py313"]
+[tool.black]
+line-length = 79
+target-version = ["py313"]
 
-# [tool.mypy]
-# python_version = "3.13"
-# warn_return_any = true
-# warn_unused_configs = true
-# disallow_untyped_defs = true
-# mypy_path           = ["pstuts_rag/pstuts_rag"]
-# namespace_packages  = true
-# explicit_package_bases = true
+[tool.mypy]
+python_version = "3.13"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+mypy_path           = ["pstuts_rag/pstuts_rag"]
+namespace_packages  = true
+explicit_package_bases = true
 
-# [tool.flake8]
-# application-import-names = "pstuts_rag"
-# extend-ignore = "E203,W503"
+[tool.flake8]
+application-import-names = "pstuts_rag"
+extend-ignore = "E203,W503"
 
-# [tool.pylint.MASTER]
-# load-plugins      = "pylint_venv"          # optional but handy
-# source-roots      = "pstuts_rag"
-# extension-pkg-allow-list = "numpy, torch"  # compiled deps that astroid cannot parse
+[tool.pylint.MASTER]
+load-plugins      = "pylint_venv"          # optional but handy
+source-roots      = "pstuts_rag"
+extension-pkg-allow-list = "numpy, torch"  # compiled deps that astroid cannot parse
 
-# [tool.pylint.TYPECHECK]
-# ignored-modules   = "pkg_resources"        # suppress noisy vendored imports
+[tool.pylint.TYPECHECK]
+ignored-modules   = "pkg_resources"        # suppress noisy vendored imports

uv.lock

@@ -235,6 +235,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764 },
 ]
 
+[[package]]
+name = "black"
+version = "25.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "mypy-extensions" },
+    { name = "packaging" },
+    { name = "pathspec" },
+    { name = "platformdirs" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/94/49/26a7b0f3f35da4b5a65f081943b7bcd22d7002f5f0fb8098ec1ff21cb6ef/black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666", size = 649449 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/98/87/0edf98916640efa5d0696e1abb0a8357b52e69e82322628f25bf14d263d1/black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f", size = 1650673 },
+    { url = "https://files.pythonhosted.org/packages/52/e5/f7bf17207cf87fa6e9b676576749c6b6ed0d70f179a3d812c997870291c3/black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3", size = 1453190 },
+    { url = "https://files.pythonhosted.org/packages/e3/ee/adda3d46d4a9120772fae6de454c8495603c37c4c3b9c60f25b1ab6401fe/black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171", size = 1782926 },
+    { url = "https://files.pythonhosted.org/packages/cc/64/94eb5f45dcb997d2082f097a3944cfc7fe87e071907f677e80788a2d7b7a/black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18", size = 1442613 },
+    { url = "https://files.pythonhosted.org/packages/09/71/54e999902aed72baf26bca0d50781b01838251a462612966e9fc4891eadd/black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717", size = 207646 },
+]
+
 [[package]]
 name = "bleach"
 version = "6.2.0"
@@ -613,6 +633,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970 },
 ]
 
+[[package]]
+name = "flake8"
+version = "7.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mccabe" },
+    { name = "pycodestyle" },
+    { name = "pyflakes" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e7/c4/5842fc9fc94584c455543540af62fd9900faade32511fab650e9891ec225/flake8-7.2.0.tar.gz", hash = "sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426", size = 48177 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/83/5c/0627be4c9976d56b1217cb5187b7504e7fd7d3503f8bfd312a04077bd4f7/flake8-7.2.0-py2.py3-none-any.whl", hash = "sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343", size = 57786 },
+]
+
 [[package]]
 name = "fqdn"
 version = "1.5.1"
@@ -885,6 +919,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/79/9d/0fb148dc4d6fa4a7dd1d8378168d9b4cd8d4560a6fbf6f0121c5fc34eb68/importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e", size = 26971 },
 ]
 
+[[package]]
+name = "iniconfig"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 },
+]
+
 [[package]]
 name = "ipykernel"
 version = "6.29.5"
@@ -970,6 +1013,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321 },
 ]
 
+[[package]]
+name = "isort"
+version = "6.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b8/21/1e2a441f74a653a144224d7d21afe8f4169e6c7c20bb13aec3a2dc3815e0/isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450", size = 821955 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/11/114d0a5f4dabbdcedc1125dee0888514c3c3b16d3e9facad87ed96fad97c/isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615", size = 94186 },
+]
+
 [[package]]
 name = "jedi"
 version = "0.19.2"
@@ -1615,6 +1667,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899 },
 ]
 
+[[package]]
+name = "mccabe"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350 },
+]
+
 [[package]]
 name = "mistune"
 version = "3.1.3"
@@ -1683,6 +1744,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351 },
 ]
 
+[[package]]
+name = "mypy"
+version = "1.15.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mypy-extensions" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ce/43/d5e49a86afa64bd3839ea0d5b9c7103487007d728e1293f52525d6d5486a/mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43", size = 3239717 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6a/9b/fd2e05d6ffff24d912f150b87db9e364fa8282045c875654ce7e32fffa66/mypy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93faf3fdb04768d44bf28693293f3904bbb555d076b781ad2530214ee53e3445", size = 10788592 },
+    { url = "https://files.pythonhosted.org/packages/74/37/b246d711c28a03ead1fd906bbc7106659aed7c089d55fe40dd58db812628/mypy-1.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:811aeccadfb730024c5d3e326b2fbe9249bb7413553f15499a4050f7c30e801d", size = 9753611 },
+    { url = "https://files.pythonhosted.org/packages/a6/ac/395808a92e10cfdac8003c3de9a2ab6dc7cde6c0d2a4df3df1b815ffd067/mypy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98b7b9b9aedb65fe628c62a6dc57f6d5088ef2dfca37903a7d9ee374d03acca5", size = 11438443 },
+    { url = "https://files.pythonhosted.org/packages/d2/8b/801aa06445d2de3895f59e476f38f3f8d610ef5d6908245f07d002676cbf/mypy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43a7682e24b4f576d93072216bf56eeff70d9140241f9edec0c104d0c515036", size = 12402541 },
+    { url = "https://files.pythonhosted.org/packages/c7/67/5a4268782eb77344cc613a4cf23540928e41f018a9a1ec4c6882baf20ab8/mypy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:baefc32840a9f00babd83251560e0ae1573e2f9d1b067719479bfb0e987c6357", size = 12494348 },
+    { url = "https://files.pythonhosted.org/packages/83/3e/57bb447f7bbbfaabf1712d96f9df142624a386d98fb026a761532526057e/mypy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b9378e2c00146c44793c98b8d5a61039a048e31f429fb0eb546d93f4b000bedf", size = 9373648 },
+    { url = "https://files.pythonhosted.org/packages/09/4e/a7d65c7322c510de2c409ff3828b03354a7c43f5a8ed458a7a131b41c7b9/mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e", size = 2221777 },
+]
+
 [[package]]
 name = "mypy-extensions"
 version = "1.1.0"
@@ -2094,6 +2174,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650 },
 ]
 
+[[package]]
+name = "pathspec"
+version = "0.12.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 },
+]
+
 [[package]]
 name = "pexpect"
 version = "4.9.0"
@@ -2124,6 +2213,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567 },
 ]
 
+[[package]]
+name = "pluggy"
+version = "1.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 },
+]
+
 [[package]]
 name = "portalocker"
 version = "2.10.1"
@@ -2224,10 +2322,12 @@ dependencies = [
     { name = "httpx" },
     { name = "ipykernel" },
     { name = "ipywidgets" },
+    { name = "isort" },
     { name = "jq" },
     { name = "jupyter" },
     { name = "langchain" },
     { name = "langchain-community" },
+    { name = "langchain-core" },
     { name = "langchain-experimental" },
     { name = "langchain-openai" },
     { name = "langchain-qdrant" },
@@ -2236,6 +2336,7 @@ dependencies = [
     { name = "openai" },
     { name = "pip" },
     { name = "pydantic" },
+    { name = "pylint-venv" },
     { name = "python-multipart" },
     { name = "ragas" },
     { name = "requests" },
@@ -2247,27 +2348,42 @@ dependencies = [
     { name = "websockets" },
 ]
 
+[package.optional-dependencies]
+dev = [
+    { name = "black" },
+    { name = "flake8" },
+    { name = "mypy" },
+    { name = "pytest" },
+]
+
 [package.metadata]
 requires-dist = [
     { name = "aiohttp", specifier = ">=3.8.0" },
+    { name = "black", marker = "extra == 'dev'", specifier = ">=22.0.0" },
     { name = "chainlit", specifier = "==2.0.4" },
     { name = "dotenv", specifier = ">=0.9.9" },
     { name = "fastapi", specifier = ">=0.115.3,<0.116" },
+    { name = "flake8", marker = "extra == 'dev'", specifier = ">=4.0.0" },
     { name = "httpx", specifier = "==0.27.0" },
     { name = "ipykernel", specifier = ">=6.29.5" },
     { name = "ipywidgets", specifier = ">=8.1.7" },
+    { name = "isort", specifier = ">=6.0.1" },
     { name = "jq", specifier = ">=1.8.0" },
     { name = "jupyter", specifier = ">=1.1.1" },
     { name = "langchain", specifier = ">=0.3.25" },
     { name = "langchain-community", specifier = ">=0.3.23" },
+    { name = "langchain-core", specifier = ">=0.3.59" },
     { name = "langchain-experimental", specifier = ">=0.3.4" },
     { name = "langchain-openai" },
     { name = "langchain-qdrant", specifier = ">=0.2.0" },
     { name = "langgraph", specifier = ">=0.4.3" },
+    { name = "mypy", marker = "extra == 'dev'", specifier = ">=0.900" },
     { name = "numpy", specifier = "==2.2.2" },
     { name = "openai", specifier = "==1.59.9" },
     { name = "pip", specifier = ">=25.0.1" },
     { name = "pydantic", specifier = "==2.10.1" },
+    { name = "pylint-venv", specifier = ">=3.0.4" },
+    { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
     { name = "python-multipart", specifier = ">=0.0.18,<0.0.19" },
     { name = "ragas", specifier = ">=0.2.15" },
     { name = "requests", specifier = ">=2.31.0" },
@@ -2278,6 +2394,7 @@ requires-dist = [
     { name = "uvicorn", specifier = ">=0.25.0,<0.26.0" },
     { name = "websockets", specifier = "==14.2" },
 ]
+provides-extras = ["dev"]
 
 [[package]]
 name = "psutil"
@@ -2338,6 +2455,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/37/40/ad395740cd641869a13bcf60851296c89624662575621968dcfafabaa7f6/pyarrow-20.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:82f1ee5133bd8f49d31be1299dc07f585136679666b502540db854968576faf9", size = 25944982 },
 ]
 
+[[package]]
+name = "pycodestyle"
+version = "2.13.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/04/6e/1f4a62078e4d95d82367f24e685aef3a672abfd27d1a868068fed4ed2254/pycodestyle-2.13.0.tar.gz", hash = "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae", size = 39312 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/be/b00116df1bfb3e0bb5b45e29d604799f7b91dd861637e4d448b4e09e6a3e/pycodestyle-2.13.0-py2.py3-none-any.whl", hash = "sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9", size = 31424 },
+]
+
 [[package]]
 name = "pycparser"
 version = "2.22"
@@ -2400,6 +2526,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b6/5f/d6d641b490fd3ec2c4c13b4244d68deea3a1b970a97be64f34fb5504ff72/pydantic_settings-2.9.1-py3-none-any.whl", hash = "sha256:59b4f431b1defb26fe620c71a7d3968a710d719f5f4cdbbdb7926edeb770f6ef", size = 44356 },
 ]
 
+[[package]]
+name = "pyflakes"
+version = "3.3.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/cc/1df338bd7ed1fa7c317081dcf29bf2f01266603b301e6858856d346a12b3/pyflakes-3.3.2.tar.gz", hash = "sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b", size = 64175 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/15/40/b293a4fa769f3b02ab9e387c707c4cbdc34f073f945de0386107d4e669e6/pyflakes-3.3.2-py2.py3-none-any.whl", hash = "sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a", size = 63164 },
+]
+
 [[package]]
 name = "pygments"
 version = "2.19.1"
@@ -2418,6 +2553,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997 },
 ]
 
+[[package]]
+name = "pylint-venv"
+version = "3.0.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/48/ac/88e53ad1362392e68a36a4f08ca82c76e6cb6321d8ecba901cca5a70dcc1/pylint_venv-3.0.4.tar.gz", hash = "sha256:4c71c2ad14fc0549699440bcc460994ffddc1481b9078404b4c43d2806ef0f59", size = 5772 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a6/10/040e1928236e3d34b26639e3427df88c7249a85aadc621cea2158589b4f8/pylint_venv-3.0.4-py3-none-any.whl", hash = "sha256:31006a3df398f58f962c9e5620e756b284e8b2bc490594ce5ee5da41920cb32c", size = 5312 },
+]
+
 [[package]]
 name = "pypdf"
 version = "5.5.0"
@@ -2427,6 +2571,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a1/4e/931b90b51e3ebc69699be926b3d5bfdabae2d9c84337fd0c9fb98adbf70c/pypdf-5.5.0-py3-none-any.whl", hash = "sha256:2f61f2d32dde00471cd70b8977f98960c64e84dd5ba0d070e953fcb4da0b2a73", size = 303371 },
 ]
 
+[[package]]
+name = "pytest"
+version = "8.3.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "iniconfig" },
+    { name = "packaging" },
+    { name = "pluggy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634 },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"