{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "28590cb1", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import time\n", "import logging\n", "import matplotlib.pyplot as plt\n", "from typing import List, Dict\n", "from dataclasses import dataclass, field\n", "from datetime import datetime\n", "from datasets import load_dataset\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 2, "id": "d62f37bb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: langchain in d:\\Documents\\anaconda\\Lib\\site-packages (1.2.10)Note: you may need to restart the kernel to use updated packages.\n", "\n", "Requirement already satisfied: langchain-community in d:\\Documents\\anaconda\\Lib\\site-packages (0.4.1)\n", "Requirement already satisfied: chromadb in d:\\Documents\\anaconda\\Lib\\site-packages (1.5.2)\n", "Requirement already satisfied: langchain-core<2.0.0,>=1.2.10 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (1.2.17)\n", "Requirement already satisfied: langgraph<1.1.0,>=1.0.8 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (1.0.10)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (2.10.3)\n", "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (1.33)\n", "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (0.7.11)\n", "Requirement already satisfied: packaging>=23.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (24.2)\n", "Requirement already satisfied: pyyaml<7.0.0,>=5.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (6.0.2)\n", "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (9.0.0)\n", "Requirement already satisfied: typing-extensions<5.0.0,>=4.7.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (4.12.2)\n", "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (0.14.1)\n", "Requirement already satisfied: jsonpointer>=1.9 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2.0.0,>=1.2.10->langchain) (2.1)\n", "Requirement already satisfied: langgraph-checkpoint<5.0.0,>=2.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (4.0.1)\n", "Requirement already satisfied: langgraph-prebuilt<1.1.0,>=1.0.8 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (1.0.8)\n", "Requirement already satisfied: langgraph-sdk<0.4.0,>=0.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (0.3.9)\n", "Requirement already satisfied: xxhash>=3.5.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (3.6.0)\n", "Requirement already satisfied: ormsgpack>=1.12.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph-checkpoint<5.0.0,>=2.1.0->langgraph<1.1.0,>=1.0.8->langchain) (1.12.2)\n", "Requirement already satisfied: httpx>=0.25.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (0.28.1)\n", "Requirement already satisfied: orjson>=3.11.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (3.11.7)\n", "Requirement already satisfied: requests-toolbelt>=1.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (1.0.0)\n", "Requirement already satisfied: requests>=2.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (2.32.5)\n", "Requirement already satisfied: zstandard>=0.23.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (0.23.0)\n", "Requirement already satisfied: anyio in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (4.7.0)\n", "Requirement already satisfied: certifi in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (2025.8.3)\n", "Requirement already satisfied: httpcore==1.* in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (1.0.9)\n", "Requirement already satisfied: idna in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (3.7)\n", "Requirement already satisfied: h11>=0.16 in d:\\Documents\\anaconda\\Lib\\site-packages (from httpcore==1.*->httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (0.16.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.27.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.27.1)\n", "Requirement already satisfied: langchain-classic<2.0.0,>=1.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (1.0.1)\n", "Requirement already satisfied: SQLAlchemy<3.0.0,>=1.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (2.0.39)\n", "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (3.11.10)\n", "Requirement already satisfied: dataclasses-json<0.7.0,>=0.6.7 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (0.6.7)\n", "Requirement already satisfied: pydantic-settings<3.0.0,>=2.10.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (2.13.1)\n", "Requirement already satisfied: httpx-sse<1.0.0,>=0.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (0.4.3)\n", "Requirement already satisfied: numpy>=2.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-community) (2.1.3)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (2.4.4)\n", "Requirement already satisfied: aiosignal>=1.1.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.2.0)\n", "Requirement already satisfied: attrs>=17.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (24.3.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (6.1.0)\n", "Requirement already satisfied: propcache>=0.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (0.3.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain-community) (1.18.0)\n", "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from dataclasses-json<0.7.0,>=0.6.7->langchain-community) (3.26.2)\n", "Requirement already satisfied: typing-inspect<1,>=0.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from dataclasses-json<0.7.0,>=0.6.7->langchain-community) (0.9.0)\n", "Requirement already satisfied: langchain-text-splitters<2.0.0,>=1.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-classic<2.0.0,>=1.0.0->langchain-community) (1.1.1)\n", "Requirement already satisfied: python-dotenv>=0.21.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic-settings<3.0.0,>=2.10.1->langchain-community) (1.1.0)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic-settings<3.0.0,>=2.10.1->langchain-community) (0.4.2)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (3.3.2)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (2.3.0)\n", "Requirement already satisfied: greenlet!=0.4.17 in d:\\Documents\\anaconda\\Lib\\site-packages (from SQLAlchemy<3.0.0,>=1.4.0->langchain-community) (3.1.1)\n", "Requirement already satisfied: mypy-extensions>=0.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7.0,>=0.6.7->langchain-community) (1.0.0)\n", "Requirement already satisfied: build>=1.0.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.4.0)\n", "Requirement already satisfied: pybase64>=1.4.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.4.3)\n", "Requirement already satisfied: uvicorn>=0.18.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.41.0)\n", "Requirement already satisfied: posthog<6.0.0,>=2.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (5.4.0)\n", "Requirement already satisfied: onnxruntime>=1.14.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.24.2)\n", "Requirement already satisfied: opentelemetry-api>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: tokenizers>=0.13.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.22.2)\n", "Requirement already satisfied: pypika>=0.48.9 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.51.1)\n", "Requirement already satisfied: tqdm>=4.65.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (4.67.1)\n", "Requirement already satisfied: overrides>=7.3.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (7.4.0)\n", "Requirement already satisfied: importlib-resources in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (6.5.2)\n", "Requirement already satisfied: grpcio>=1.58.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.76.0)\n", "Requirement already satisfied: bcrypt>=4.0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (4.3.0)\n", "Requirement already satisfied: typer>=0.9.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.9.0)\n", "Requirement already satisfied: kubernetes>=28.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (35.0.0)\n", "Requirement already satisfied: mmh3>=4.0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (5.2.0)\n", "Requirement already satisfied: rich>=10.11.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (13.9.4)\n", "Requirement already satisfied: jsonschema>=4.19.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (4.23.0)\n", "Requirement already satisfied: six>=1.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (1.17.0)\n", "Requirement already satisfied: python-dateutil>=2.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (2.9.0.post0)\n", "Requirement already satisfied: backoff>=1.10.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (2.2.1)\n", "Requirement already satisfied: distro>=1.5.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (1.9.0)\n", "Requirement already satisfied: pyproject_hooks in d:\\Documents\\anaconda\\Lib\\site-packages (from build>=1.0.3->chromadb) (1.2.0)\n", "Requirement already satisfied: colorama in d:\\Documents\\anaconda\\Lib\\site-packages (from build>=1.0.3->chromadb) (0.4.6)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (2023.7.1)\n", "Requirement already satisfied: referencing>=0.28.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (0.30.2)\n", "Requirement already satisfied: rpds-py>=0.7.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (0.22.3)\n", "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n", "Requirement already satisfied: requests-oauthlib in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n", "Requirement already satisfied: durationpy>=0.7 in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (0.10)\n", "Requirement already satisfied: flatbuffers in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (25.9.23)\n", "Requirement already satisfied: protobuf in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (6.33.5)\n", "Requirement already satisfied: sympy in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.3)\n", "Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-api>=1.2.0->chromadb) (8.5.0)\n", "Requirement already satisfied: zipp>=3.20 in d:\\Documents\\anaconda\\Lib\\site-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.21.0)\n", "Requirement already satisfied: googleapis-common-protos~=1.57 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.72.0)\n", "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.39.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-proto==1.39.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-semantic-conventions==0.60b1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-sdk>=1.2.0->chromadb) (0.60b1)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from rich>=10.11.0->chromadb) (2.2.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from rich>=10.11.0->chromadb) (2.19.1)\n", "Requirement already satisfied: mdurl~=0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->chromadb) (0.1.0)\n", "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from tokenizers>=0.13.2->chromadb) (1.4.1)\n", "Requirement already satisfied: filelock in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.17.0)\n", "Requirement already satisfied: fsspec>=2023.5.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2025.3.2)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (1.2.0)\n", "Requirement already satisfied: shellingham in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (1.5.0)\n", "Requirement already satisfied: typer-slim in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (0.21.1)\n", "Requirement already satisfied: click<9.0.0,>=7.1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from typer>=0.9.0->chromadb) (8.1.8)\n", "Requirement already satisfied: httptools>=0.6.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.7.1)\n", "Requirement already satisfied: watchfiles>=0.20 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.1.1)\n", "Requirement already satisfied: websockets>=10.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (15.0.1)\n", "Requirement already satisfied: sniffio>=1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from anyio->httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (1.3.0)\n", "Requirement already satisfied: oauthlib>=3.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests-oauthlib->kubernetes>=28.1.0->chromadb) (3.3.1)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n", "Requirement already satisfied: datasets in d:\\Documents\\anaconda\\Lib\\site-packages (4.5.0)\n", "Requirement already satisfied: langchain in d:\\Documents\\anaconda\\Lib\\site-packages (1.2.10)\n", "Requirement already satisfied: chromadb in d:\\Documents\\anaconda\\Lib\\site-packages (1.5.2)\n", "Requirement already satisfied: sentence-transformers in d:\\Documents\\anaconda\\Lib\\site-packages (5.2.3)\n", "Requirement already satisfied: rank-bm25 in d:\\Documents\\anaconda\\Lib\\site-packages (0.2.2)\n", "Requirement already satisfied: pymupdf in d:\\Documents\\anaconda\\Lib\\site-packages (1.27.1)\n", "Requirement already satisfied: filelock in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (3.17.0)\n", "Requirement already satisfied: numpy>=1.17 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (2.1.3)\n", "Requirement already satisfied: pyarrow>=21.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (23.0.0)\n", "Requirement already satisfied: dill<0.4.1,>=0.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (0.4.0)\n", "Requirement already satisfied: pandas in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (2.2.3)\n", "Requirement already satisfied: requests>=2.32.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (2.32.5)\n", "Requirement already satisfied: httpx<1.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (0.28.1)\n", "Requirement already satisfied: tqdm>=4.66.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (4.67.1)\n", "Requirement already satisfied: xxhash in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (3.6.0)\n", "Requirement already satisfied: multiprocess<0.70.19 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (0.70.18)\n", "Requirement already satisfied: fsspec<=2025.10.0,>=2023.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (2025.3.2)\n", "Requirement already satisfied: huggingface-hub<2.0,>=0.25.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (1.4.1)\n", "Requirement already satisfied: packaging in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (24.2)\n", "Requirement already satisfied: pyyaml>=5.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from datasets) (6.0.2)\n", "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in d:\\Documents\\anaconda\\Lib\\site-packages (from fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (3.11.10)\n", "Requirement already satisfied: anyio in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx<1.0.0->datasets) (4.7.0)\n", "Requirement already satisfied: certifi in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx<1.0.0->datasets) (2025.8.3)\n", "Requirement already satisfied: httpcore==1.* in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx<1.0.0->datasets) (1.0.9)\n", "Requirement already satisfied: idna in d:\\Documents\\anaconda\\Lib\\site-packages (from httpx<1.0.0->datasets) (3.7)\n", "Requirement already satisfied: h11>=0.16 in d:\\Documents\\anaconda\\Lib\\site-packages (from httpcore==1.*->httpx<1.0.0->datasets) (0.16.0)\n", "Requirement already satisfied: hf-xet<2.0.0,>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.2.0)\n", "Requirement already satisfied: shellingham in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (1.5.0)\n", "Requirement already satisfied: typer-slim in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (0.21.1)\n", "Requirement already satisfied: typing-extensions>=4.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from huggingface-hub<2.0,>=0.25.0->datasets) (4.12.2)\n", "Requirement already satisfied: langchain-core<2.0.0,>=1.2.10 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (1.2.17)\n", "Requirement already satisfied: langgraph<1.1.0,>=1.0.8 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (1.0.10)\n", "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain) (2.10.3)\n", "Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (1.33)\n", "Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (0.7.11)\n", "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (9.0.0)\n", "Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langchain-core<2.0.0,>=1.2.10->langchain) (0.14.1)\n", "Requirement already satisfied: jsonpointer>=1.9 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core<2.0.0,>=1.2.10->langchain) (2.1)\n", "Requirement already satisfied: langgraph-checkpoint<5.0.0,>=2.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (4.0.1)\n", "Requirement already satisfied: langgraph-prebuilt<1.1.0,>=1.0.8 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (1.0.8)\n", "Requirement already satisfied: langgraph-sdk<0.4.0,>=0.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph<1.1.0,>=1.0.8->langchain) (0.3.9)\n", "Requirement already satisfied: ormsgpack>=1.12.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph-checkpoint<5.0.0,>=2.1.0->langgraph<1.1.0,>=1.0.8->langchain) (1.12.2)\n", "Requirement already satisfied: orjson>=3.11.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.1.0,>=1.0.8->langchain) (3.11.7)\n", "Requirement already satisfied: requests-toolbelt>=1.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (1.0.0)\n", "Requirement already satisfied: zstandard>=0.23.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core<2.0.0,>=1.2.10->langchain) (0.23.0)\n", "Requirement already satisfied: annotated-types>=0.6.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.6.0)\n", "Requirement already satisfied: pydantic-core==2.27.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.27.1)\n", "Requirement already satisfied: build>=1.0.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.4.0)\n", "Requirement already satisfied: pybase64>=1.4.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.4.3)\n", "Requirement already satisfied: uvicorn>=0.18.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.41.0)\n", "Requirement already satisfied: posthog<6.0.0,>=2.4.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (5.4.0)\n", "Requirement already satisfied: onnxruntime>=1.14.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.24.2)\n", "Requirement already satisfied: opentelemetry-api>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.39.1)\n", "Requirement already satisfied: tokenizers>=0.13.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.22.2)\n", "Requirement already satisfied: pypika>=0.48.9 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.51.1)\n", "Requirement already satisfied: overrides>=7.3.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (7.4.0)\n", "Requirement already satisfied: importlib-resources in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (6.5.2)\n", "Requirement already satisfied: grpcio>=1.58.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (1.76.0)\n", "Requirement already satisfied: bcrypt>=4.0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (4.3.0)\n", "Requirement already satisfied: typer>=0.9.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (0.9.0)\n", "Requirement already satisfied: kubernetes>=28.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (35.0.0)\n", "Requirement already satisfied: mmh3>=4.0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (5.2.0)\n", "Requirement already satisfied: rich>=10.11.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (13.9.4)\n", "Requirement already satisfied: jsonschema>=4.19.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from chromadb) (4.23.0)\n", "Requirement already satisfied: six>=1.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (1.17.0)\n", "Requirement already satisfied: python-dateutil>=2.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (2.9.0.post0)\n", "Requirement already satisfied: backoff>=1.10.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (2.2.1)\n", "Requirement already satisfied: distro>=1.5.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from posthog<6.0.0,>=2.4.0->chromadb) (1.9.0)\n", "Requirement already satisfied: charset_normalizer<4,>=2 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests>=2.32.2->datasets) (3.3.2)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests>=2.32.2->datasets) (2.3.0)\n", "Requirement already satisfied: transformers<6.0.0,>=4.41.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from sentence-transformers) (5.1.0)\n", "Requirement already satisfied: torch>=1.11.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from sentence-transformers) (2.7.1+cu118)\n", "Requirement already satisfied: scikit-learn in d:\\Documents\\anaconda\\Lib\\site-packages (from sentence-transformers) (1.6.1)\n", "Requirement already satisfied: scipy in d:\\Documents\\anaconda\\Lib\\site-packages (from sentence-transformers) (1.15.3)\n", "Requirement already satisfied: regex!=2019.12.17 in d:\\Documents\\anaconda\\Lib\\site-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (2024.11.6)\n", "Requirement already satisfied: safetensors>=0.4.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.7.0)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (2.4.4)\n", "Requirement already satisfied: aiosignal>=1.1.2 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.2.0)\n", "Requirement already satisfied: attrs>=17.3.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (24.3.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (6.1.0)\n", "Requirement already satisfied: propcache>=0.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (0.3.1)\n", "Requirement already satisfied: yarl<2.0,>=1.17.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]<=2025.10.0,>=2023.1.0->datasets) (1.18.0)\n", "Requirement already satisfied: pyproject_hooks in d:\\Documents\\anaconda\\Lib\\site-packages (from build>=1.0.3->chromadb) (1.2.0)\n", "Requirement already satisfied: colorama in d:\\Documents\\anaconda\\Lib\\site-packages (from build>=1.0.3->chromadb) (0.4.6)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (2023.7.1)\n", "Requirement already satisfied: referencing>=0.28.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (0.30.2)\n", "Requirement already satisfied: rpds-py>=0.7.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from jsonschema>=4.19.0->chromadb) (0.22.3)\n", "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n", "Requirement already satisfied: requests-oauthlib in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n", "Requirement already satisfied: durationpy>=0.7 in d:\\Documents\\anaconda\\Lib\\site-packages (from kubernetes>=28.1.0->chromadb) (0.10)\n", "Requirement already satisfied: flatbuffers in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (25.9.23)\n", "Requirement already satisfied: protobuf in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (6.33.5)\n", "Requirement already satisfied: sympy in d:\\Documents\\anaconda\\Lib\\site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.3)\n", "Requirement already satisfied: importlib-metadata<8.8.0,>=6.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-api>=1.2.0->chromadb) (8.5.0)\n", "Requirement already satisfied: zipp>=3.20 in d:\\Documents\\anaconda\\Lib\\site-packages (from importlib-metadata<8.8.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.21.0)\n", "Requirement already satisfied: googleapis-common-protos~=1.57 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.72.0)\n", "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.39.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-proto==1.39.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.39.1)\n", "Requirement already satisfied: opentelemetry-semantic-conventions==0.60b1 in d:\\Documents\\anaconda\\Lib\\site-packages (from opentelemetry-sdk>=1.2.0->chromadb) (0.60b1)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from rich>=10.11.0->chromadb) (2.2.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from rich>=10.11.0->chromadb) (2.19.1)\n", "Requirement already satisfied: mdurl~=0.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->chromadb) (0.1.0)\n", "Requirement already satisfied: networkx in d:\\Documents\\anaconda\\Lib\\site-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n", "Requirement already satisfied: jinja2 in d:\\Documents\\anaconda\\Lib\\site-packages (from torch>=1.11.0->sentence-transformers) (3.1.6)\n", "Requirement already satisfied: setuptools in d:\\Documents\\anaconda\\Lib\\site-packages (from torch>=1.11.0->sentence-transformers) (72.1.0)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n", "Requirement already satisfied: click<9.0.0,>=7.1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from typer>=0.9.0->chromadb) (8.1.8)\n", "Requirement already satisfied: httptools>=0.6.3 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.7.1)\n", "Requirement already satisfied: python-dotenv>=0.13 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.1.0)\n", "Requirement already satisfied: watchfiles>=0.20 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.1.1)\n", "Requirement already satisfied: websockets>=10.4 in d:\\Documents\\anaconda\\Lib\\site-packages (from uvicorn[standard]>=0.18.3->chromadb) (15.0.1)\n", "Requirement already satisfied: sniffio>=1.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from anyio->httpx<1.0.0->datasets) (1.3.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.2)\n", "Requirement already satisfied: pytz>=2020.1 in d:\\Documents\\anaconda\\Lib\\site-packages (from pandas->datasets) (2024.1)\n", "Requirement already satisfied: tzdata>=2022.7 in d:\\Documents\\anaconda\\Lib\\site-packages (from pandas->datasets) (2025.2)\n", "Requirement already satisfied: oauthlib>=3.0.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from requests-oauthlib->kubernetes>=28.1.0->chromadb) (3.3.1)\n", "Requirement already satisfied: joblib>=1.2.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from scikit-learn->sentence-transformers) (1.4.2)\n", "Requirement already satisfied: threadpoolctl>=3.1.0 in d:\\Documents\\anaconda\\Lib\\site-packages (from scikit-learn->sentence-transformers) (3.5.0)\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } ], "source": [ "%pip install langchain langchain-community chromadb\n", "%pip install datasets langchain chromadb sentence-transformers rank-bm25 pymupdf" ] }, { "cell_type": "code", "execution_count": 3, "id": "4a39b4a8", "metadata": {}, "outputs": [], "source": [ "# Configuration\n", "CONFIG = {\n", " \"embedding_model\": \"sentence-transformers/all-MiniLM-L6-v2\",\n", " \"dataset_name\": \"MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1\",\n", " \"dense_weight\": 0.7, # RRF Weight for Semantic search\n", " \"top_k\": 5}\n", "\n", "logging.basicConfig(level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\")\n", "logger = logging.getLogger(__name__)" ] }, { "cell_type": "markdown", "id": "87ead447", "metadata": {}, "source": [ "### Data Loading" ] }, { "cell_type": "code", "execution_count": 4, "id": "f7e8d163", "metadata": {}, "outputs": [], "source": [ "def load_markrai_dataset():\n", " \"\"\"Loads the 8.5k research corpus nodes and 520 QA test cases.\"\"\"\n", " logger.info(\"Fetching corpus and QA subsets from Hugging Face...\")\n", " corpus_ds = load_dataset(CONFIG['dataset_name'], \"corpus\", split=\"train\")\n", " qa_ds = load_dataset(CONFIG['dataset_name'], \"qa\", split=\"train\")\n", " \n", " docs = [{\"content\": r[\"contents\"], \"id\": r[\"doc_id\"]} for r in corpus_ds]\n", " test_set = [{\"question\": r[\"query\"], \"ground_truth\": r[\"generation_gt\"][0]} for r in qa_ds]\n", " \n", " return docs, test_set\n" ] }, { "cell_type": "markdown", "id": "578f5fbe", "metadata": {}, "source": [ "### Hybrid Search Engine Architecture" ] }, { "cell_type": "code", "execution_count": 5, "id": "dcaacc39", "metadata": {}, "outputs": [], "source": [ "class HybridSearchEngine:\n", " \"\"\"Combines Dense (Embeddings) and Sparse (BM25) for precision.\"\"\"\n", " def __init__(self, documents: List[Dict]):\n", " from sentence_transformers import SentenceTransformer\n", " from rank_bm25 import BM25Okapi\n", " \n", " self.docs = documents\n", " self.model = SentenceTransformer(CONFIG[\"embedding_model\"])\n", " self.embeddings = self.model.encode([d[\"content\"] for d in documents], normalize_embeddings=True)\n", " \n", " tokenized_corpus = [d[\"content\"].lower().split() for d in documents]\n", " self.bm25 = BM25Okapi(tokenized_corpus)\n", "\n", " def search(self, query: str, top_k: int = 5) -> List[Dict]:\n", " # Dense Rank\n", " q_emb = self.model.encode([query], normalize_embeddings=True)[0]\n", " dense_ranks = np.argsort(-np.dot(self.embeddings, q_emb))\n", " \n", " # Sparse Rank\n", " bm25_ranks = np.argsort(-self.bm25.get_scores(query.lower().split()))\n", " \n", " # Reciprocal Rank Fusion (RRF)\n", " k = 60\n", " scores = np.zeros(len(self.docs))\n", " for rank, idx in enumerate(dense_ranks):\n", " scores[idx] += CONFIG[\"dense_weight\"] * (1 / (rank + k))\n", " for rank, idx in enumerate(bm25_ranks):\n", " scores[idx] += (1 - CONFIG[\"dense_weight\"]) * (1 / (rank + k))\n", " \n", " return [dict(self.docs[i], rrf_score=scores[i]) for i in np.argsort(-scores)[:top_k]]\n" ] }, { "cell_type": "markdown", "id": "ee66562f", "metadata": {}, "source": [ "### RAG Pipeline and Monitoring" ] }, { "cell_type": "code", "execution_count": 6, "id": "c32845b5", "metadata": {}, "outputs": [], "source": [ "@dataclass\n", "class RAGResponse:\n", " query: str\n", " latency_ms: float\n", " recall_score: float\n", "\n", "class RAGPipeline:\n", " def __init__(self, engine: HybridSearchEngine):\n", " self.engine = engine\n", "\n", " def query(self, user_query: str, ground_truth: str) -> RAGResponse:\n", " t0 = time.time()\n", " results = self.engine.search(user_query, top_k=CONFIG[\"top_k\"])\n", " latency = (time.time() - t0) * 1000\n", " \n", " # Calculate Context Recall: Does retrieved text contain key GT words?\n", " context = \" \".join([r[\"content\"] for r in results]).lower()\n", " gt_tokens = set(ground_truth.lower().split())\n", " recall = sum(1 for t in gt_tokens if t in context) / len(gt_tokens) if gt_tokens else 0\n", " \n", " return RAGResponse(query=user_query, latency_ms=latency, recall_score=recall)\n" ] }, { "cell_type": "markdown", "id": "8867ea0d", "metadata": {}, "source": [ "### Dashboard and Analysis" ] }, { "cell_type": "code", "execution_count": 7, "id": "7daf27ad", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2026-03-17 15:17:20,961 INFO Fetching corpus and QA subsets from Hugging Face...\n", "2026-03-17 15:17:22,978 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/main/README.md \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:22,993 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/fc10b797869aaa4f8c7f007082d962c507b16fac/README.md \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:23,256 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/AutoRAG-evaluation-2024-LLM-paper-v1.py \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:24,042 INFO HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1.py \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:24,309 INFO HTTP Request: GET https://huggingface.co/api/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/revision/fc10b797869aaa4f8c7f007082d962c507b16fac \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:25,016 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/.huggingface.yaml \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:26,595 INFO HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1 \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:27,254 INFO HTTP Request: GET https://huggingface.co/api/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/tree/fc10b797869aaa4f8c7f007082d962c507b16fac/qa?recursive=true&expand=false \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:27,500 INFO HTTP Request: GET https://huggingface.co/api/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/tree/fc10b797869aaa4f8c7f007082d962c507b16fac?recursive=false&expand=false \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:27,814 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/dataset_infos.json \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:28,083 INFO HTTP Request: GET https://huggingface.co/api/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/tree/fc10b797869aaa4f8c7f007082d962c507b16fac/corpus?recursive=true&expand=false \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:30,202 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/main/README.md \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:31,622 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/fc10b797869aaa4f8c7f007082d962c507b16fac/README.md \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:31,875 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/AutoRAG-evaluation-2024-LLM-paper-v1.py \"HTTP/1.1 404 Not Found\"\n", "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n", "2026-03-17 15:17:31,877 WARNING Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.\n", "2026-03-17 15:17:32,673 INFO HTTP Request: HEAD https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1.py \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:32,926 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/.huggingface.yaml \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:33,196 INFO HTTP Request: GET https://datasets-server.huggingface.co/info?dataset=MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1 \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:33,424 INFO HTTP Request: HEAD https://huggingface.co/datasets/MarkrAI/AutoRAG-evaluation-2024-LLM-paper-v1/resolve/fc10b797869aaa4f8c7f007082d962c507b16fac/dataset_infos.json \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:17:53,472 INFO Use pytorch device_name: cuda:0\n", "2026-03-17 15:17:53,477 INFO Load pretrained SentenceTransformer: sentence-transformers/all-MiniLM-L6-v2\n", "2026-03-17 15:17:53,783 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:54,055 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/modules.json \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:55,948 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config_sentence_transformers.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:55,958 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/config_sentence_transformers.json \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:56,218 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config_sentence_transformers.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:56,228 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/config_sentence_transformers.json \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:56,498 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/README.md \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:56,758 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/README.md \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:57,027 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/modules.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:57,038 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/modules.json \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:17:57,547 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/sentence_bert_config.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:17:58,237 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/sentence_bert_config.json \"HTTP/1.1 200 OK\"\n", "2026-03-17 15:18:01,387 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/adapter_config.json \"HTTP/1.1 404 Not Found\"\n", "2026-03-17 15:18:01,648 INFO HTTP Request: HEAD https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config.json \"HTTP/1.1 307 Temporary Redirect\"\n", "2026-03-17 15:18:01,660 INFO HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/sentence-transformers/all-MiniLM-L6-v2/c9745ed1d9f207416be6d2e6f8de32d1f16199bf/config.json \"HTTP/1.1 200 OK\"\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "75f3125ce1b5483998956554a4bdbc69", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading weights: 0%| | 0/103 [00:00