Delete Rag_milvus.ipynb

Rag_milvus.ipynb

@@ -1,571 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "bf597549",
-   "metadata": {},
-   "source": [
-    "PRIMERO PREPARAMOS TODAS NUESTRAS FUNCIONES PARA PODER SER CONVOCADAS LUEGO."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7968949c",
-   "metadata": {},
-   "source": [
-    "Instalamos "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "9a192af6",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Collecting qdrant-client\n",
-      "  Obtaining dependency information for qdrant-client from https://files.pythonhosted.org/packages/e4/52/f49b0aa96253010f57cf80315edecec4f469e7a39c1ed92bf727fa290e57/qdrant_client-1.14.2-py3-none-any.whl.metadata\n",
-      "  Downloading qdrant_client-1.14.2-py3-none-any.whl.metadata (10 kB)\n",
-      "Requirement already satisfied: transformers in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (4.51.3)\n",
-      "Requirement already satisfied: torch in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (2.7.0)\n",
-      "Requirement already satisfied: langchain in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (0.3.24)\n",
-      "Collecting pymupdf\n",
-      "  Obtaining dependency information for pymupdf from https://files.pythonhosted.org/packages/71/c2/a9059607f80dcaf2392f991748cfc53456820392c0220cff02572653512a/pymupdf-1.25.5-cp39-abi3-win_amd64.whl.metadata\n",
-      "  Downloading pymupdf-1.25.5-cp39-abi3-win_amd64.whl.metadata (3.4 kB)\n",
-      "Collecting grpcio>=1.41.0 (from qdrant-client)\n",
-      "  Obtaining dependency information for grpcio>=1.41.0 from https://files.pythonhosted.org/packages/ee/3f/cf92e7e62ccb8dbdf977499547dfc27133124d6467d3a7d23775bcecb0f9/grpcio-1.71.0-cp311-cp311-win_amd64.whl.metadata\n",
-      "  Using cached grpcio-1.71.0-cp311-cp311-win_amd64.whl.metadata (4.0 kB)\n",
-      "Requirement already satisfied: httpx[http2]>=0.20.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (0.28.1)\n",
-      "Requirement already satisfied: numpy>=1.21 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.2.5)\n",
-      "Collecting portalocker<3.0.0,>=2.7.0 (from qdrant-client)\n",
-      "  Obtaining dependency information for portalocker<3.0.0,>=2.7.0 from https://files.pythonhosted.org/packages/9b/fb/a70a4214956182e0d7a9099ab17d50bfcba1056188e9b14f35b9e2b62a0d/portalocker-2.10.1-py3-none-any.whl.metadata\n",
-      "  Using cached portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n",
-      "Collecting protobuf>=3.20.0 (from qdrant-client)\n",
-      "  Obtaining dependency information for protobuf>=3.20.0 from https://files.pythonhosted.org/packages/97/e9/7b9f1b259d509aef2b833c29a1f3c39185e2bf21c9c1be1cd11c22cb2149/protobuf-6.30.2-cp310-abi3-win_amd64.whl.metadata\n",
-      "  Downloading protobuf-6.30.2-cp310-abi3-win_amd64.whl.metadata (593 bytes)\n",
-      "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.11.4)\n",
-      "Requirement already satisfied: urllib3<3,>=1.26.14 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.4.0)\n",
-      "Requirement already satisfied: filelock in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (3.18.0)\n",
-      "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.30.2)\n",
-      "Requirement already satisfied: packaging>=20.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (24.2)\n",
-      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (6.0.2)\n",
-      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (2024.11.6)\n",
-      "Requirement already satisfied: requests in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (2.32.3)\n",
-      "Requirement already satisfied: tokenizers<0.22,>=0.21 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.21.1)\n",
-      "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.5.3)\n",
-      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (4.67.1)\n",
-      "Requirement already satisfied: typing-extensions>=4.10.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (4.13.2)\n",
-      "Requirement already satisfied: sympy>=1.13.3 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (1.14.0)\n",
-      "Requirement already satisfied: networkx in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (3.4.2)\n",
-      "Requirement already satisfied: jinja2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (3.1.6)\n",
-      "Requirement already satisfied: fsspec in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (2025.3.2)\n",
-      "Requirement already satisfied: langchain-core<1.0.0,>=0.3.55 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.56)\n",
-      "Requirement already satisfied: langchain-text-splitters<1.0.0,>=0.3.8 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.8)\n",
-      "Requirement already satisfied: langsmith<0.4,>=0.1.17 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.39)\n",
-      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (2.0.40)\n",
-      "Requirement already satisfied: anyio in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (4.9.0)\n",
-      "Requirement already satisfied: certifi in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (2025.4.26)\n",
-      "Requirement already satisfied: httpcore==1.* in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (1.0.9)\n",
-      "Requirement already satisfied: idna in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (3.10)\n",
-      "Collecting h2<5,>=3 (from httpx[http2]>=0.20.0->qdrant-client)\n",
-      "  Obtaining dependency information for h2<5,>=3 from https://files.pythonhosted.org/packages/d0/9e/984486f2d0a0bd2b024bf4bc1c62688fcafa9e61991f041fb0e2def4a982/h2-4.2.0-py3-none-any.whl.metadata\n",
-      "  Using cached h2-4.2.0-py3-none-any.whl.metadata (5.1 kB)\n",
-      "Requirement already satisfied: h11>=0.16 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpcore==1.*->httpx[http2]>=0.20.0->qdrant-client) (0.16.0)\n",
-      "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain-core<1.0.0,>=0.3.55->langchain) (9.1.2)\n",
-      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain-core<1.0.0,>=0.3.55->langchain) (1.33)\n",
-      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (3.10.18)\n",
-      "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (1.0.0)\n",
-      "Requirement already satisfied: zstandard<0.24.0,>=0.23.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (0.23.0)\n",
-      "Requirement already satisfied: pywin32>=226 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from portalocker<3.0.0,>=2.7.0->qdrant-client) (310)\n",
-      "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.7.0)\n",
-      "Requirement already satisfied: pydantic-core==2.33.2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (2.33.2)\n",
-      "Requirement already satisfied: typing-inspection>=0.4.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.4.0)\n",
-      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from requests->transformers) (3.4.1)\n",
-      "Requirement already satisfied: greenlet>=1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.2.1)\n",
-      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from sympy>=1.13.3->torch) (1.3.0)\n",
-      "Requirement already satisfied: colorama in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n",
-      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from jinja2->torch) (3.0.2)\n",
-      "Collecting hyperframe<7,>=6.1 (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client)\n",
-      "  Obtaining dependency information for hyperframe<7,>=6.1 from https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl.metadata\n",
-      "  Using cached hyperframe-6.1.0-py3-none-any.whl.metadata (4.3 kB)\n",
-      "Collecting hpack<5,>=4.1 (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client)\n",
-      "  Obtaining dependency information for hpack<5,>=4.1 from https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl.metadata\n",
-      "  Using cached hpack-4.1.0-py3-none-any.whl.metadata (4.6 kB)\n",
-      "Requirement already satisfied: jsonpointer>=1.9 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<1.0.0,>=0.3.55->langchain) (3.0.0)\n",
-      "Requirement already satisfied: sniffio>=1.1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from anyio->httpx[http2]>=0.20.0->qdrant-client) (1.3.1)\n",
-      "Downloading qdrant_client-1.14.2-py3-none-any.whl (327 kB)\n",
-      "   ---------------------------------------- 0.0/327.7 kB ? eta -:--:--\n",
-      "   -------------- ------------------------- 122.9/327.7 kB 3.6 MB/s eta 0:00:01\n",
-      "   ---------------------------------------- 327.7/327.7 kB 5.1 MB/s eta 0:00:00\n",
-      "Downloading pymupdf-1.25.5-cp39-abi3-win_amd64.whl (16.6 MB)\n",
-      "   ---------------------------------------- 0.0/16.6 MB ? eta -:--:--\n",
-      "   - -------------------------------------- 0.5/16.6 MB 14.4 MB/s eta 0:00:02\n",
-      "   -- ------------------------------------- 1.0/16.6 MB 12.8 MB/s eta 0:00:02\n",
-      "   ---- ----------------------------------- 1.8/16.6 MB 14.0 MB/s eta 0:00:02\n",
-      "   ------ --------------------------------- 2.5/16.6 MB 14.6 MB/s eta 0:00:01\n",
-      "   ------- -------------------------------- 3.3/16.6 MB 15.0 MB/s eta 0:00:01\n",
-      "   --------- ------------------------------ 4.0/16.6 MB 14.9 MB/s eta 0:00:01\n",
-      "   ----------- ---------------------------- 4.7/16.6 MB 15.1 MB/s eta 0:00:01\n",
-      "   ------------- -------------------------- 5.4/16.6 MB 15.1 MB/s eta 0:00:01\n",
-      "   -------------- ------------------------- 6.2/16.6 MB 15.1 MB/s eta 0:00:01\n",
-      "   ---------------- ----------------------- 6.9/16.6 MB 15.2 MB/s eta 0:00:01\n",
-      "   ------------------ --------------------- 7.6/16.6 MB 15.2 MB/s eta 0:00:01\n",
-      "   -------------------- ------------------- 8.3/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   --------------------- ------------------ 9.1/16.6 MB 15.7 MB/s eta 0:00:01\n",
-      "   ----------------------- ---------------- 9.8/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ------------------------- -------------- 10.4/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   -------------------------- ------------- 11.0/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ---------------------------- ----------- 11.8/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ------------------------------ --------- 12.5/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ------------------------------- -------- 13.2/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   --------------------------------- ------ 13.9/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ----------------------------------- ---- 14.6/16.6 MB 15.2 MB/s eta 0:00:01\n",
-      "   ------------------------------------ --- 15.3/16.6 MB 15.2 MB/s eta 0:00:01\n",
-      "   -------------------------------------- - 16.1/16.6 MB 16.0 MB/s eta 0:00:01\n",
-      "   ---------------------------------------  16.6/16.6 MB 15.6 MB/s eta 0:00:01\n",
-      "   ---------------------------------------- 16.6/16.6 MB 15.2 MB/s eta 0:00:00\n",
-      "Using cached grpcio-1.71.0-cp311-cp311-win_amd64.whl (4.3 MB)\n",
-      "Using cached portalocker-2.10.1-py3-none-any.whl (18 kB)\n",
-      "Downloading protobuf-6.30.2-cp310-abi3-win_amd64.whl (431 kB)\n",
-      "   ---------------------------------------- 0.0/431.0 kB ? eta -:--:--\n",
-      "   --------------------------------------- 431.0/431.0 kB 26.3 MB/s eta 0:00:00\n",
-      "Using cached h2-4.2.0-py3-none-any.whl (60 kB)\n",
-      "Using cached hpack-4.1.0-py3-none-any.whl (34 kB)\n",
-      "Using cached hyperframe-6.1.0-py3-none-any.whl (13 kB)\n",
-      "Installing collected packages: pymupdf, protobuf, portalocker, hyperframe, hpack, grpcio, h2, qdrant-client\n",
-      "Successfully installed grpcio-1.71.0 h2-4.2.0 hpack-4.1.0 hyperframe-6.1.0 portalocker-2.10.1 protobuf-6.30.2 pymupdf-1.25.5 qdrant-client-1.14.2\n",
-      "Note: you may need to restart the kernel to use updated packages.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "[notice] A new release of pip is available: 23.2.1 -> 25.1\n",
-      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
-     ]
-    }
-   ],
-   "source": [
-    "%pip install qdrant-client transformers torch langchain pymupdf"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a4833977",
-   "metadata": {},
-   "source": [
-    "Importamos librerias"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "id": "1684b4de",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from sentence_transformers import SentenceTransformer\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
-    "from langchain.schema import Document\n",
-    "from qdrant_client import QdrantClient\n",
-    "from qdrant_client.models import PointStruct, Distance, VectorParams\n",
-    "import fitz  # PyMuPDF"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e612e674",
-   "metadata": {},
-   "source": [
-    "Definimos funciones\n",
-    "1) Cargar los pdf por sus bloques de paginas"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "id": "5a594ed8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def pdfachunk(path, chunk_size_pages=20):\n",
-    "    doc = fitz.open(path)\n",
-    "    chunks = []\n",
-    "    for i in range(0, len(doc), chunk_size_pages):\n",
-    "        text = \"\"\n",
-    "        for page_num in range(i, min(i + chunk_size_pages, len(doc))):\n",
-    "            text += doc[page_num].get_text()\n",
-    "        chunks.append(text)\n",
-    "    doc.close()\n",
-    "    return chunks"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "59d048b7",
-   "metadata": {},
-   "source": [
-    "2) Dividir texto en chunks más pequeños con solapamiento"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "id": "bffac6eb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def split_chunks(raw_chunks, chunk_size=1024, chunk_overlap=100):\n",
-    "    docs = [Document(page_content=chunk) for chunk in raw_chunks]\n",
-    "    splitter = RecursiveCharacterTextSplitter(\n",
-    "        chunk_size=chunk_size,\n",
-    "        chunk_overlap=chunk_overlap,\n",
-    "        separators=[\"\\n\\n\", \"\\n\", \".\", \" \"]\n",
-    "    )\n",
-    "    return splitter.split_documents(docs)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "8664bf6f",
-   "metadata": {},
-   "source": [
-    "3) Generar embeddings en batch"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "35a4df0b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def generaremben(model, texts):\n",
-    "    texts = [t for t in texts if t.strip()]  # filtra vacíos\n",
-    "    if not texts:\n",
-    "        raise ValueError(\"No hay textos válidos para generar embeddings.\")\n",
-    "    return model.encode(texts, batch_size=16, show_progress_bar=True)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c28a5724",
-   "metadata": {},
-   "source": [
-    "4) Insertar los docemtos en QDRANT localemente"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "id": "3c61ddca",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def insertarenqdra(embeddings, texts, collection_name=\"pdf_chunks\"):\n",
-    "    client = QdrantClient(path=\"./qdrant_data\")  # persistente\n",
-    "\n",
-    "    dim = len(embeddings[0])\n",
-    "    client.recreate_collection(\n",
-    "        collection_name=collection_name,\n",
-    "        vectors_config=VectorParams(size=dim, distance=Distance.COSINE)\n",
-    "    )\n",
-    "\n",
-    "    points = [\n",
-    "        PointStruct(id=i, vector=embeddings[i].tolist(), payload={\"text\": texts[i]})\n",
-    "        for i in range(len(embeddings))\n",
-    "    ]\n",
-    "\n",
-    "    client.upsert(collection_name=collection_name, points=points)\n",
-    "    print(f\"✅ Insertados {len(points)} vectores en Qdrant.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "566e06c8",
-   "metadata": {},
-   "source": [
-    "5) Funcion modelo para no cargarlo siempre"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "eec86477",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def load_nv_model():\n",
-    "    return AutoModel.from_pretrained(\"nvidia/NV-Embed-v2\", trust_remote_code=True)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "08510d36",
-   "metadata": {},
-   "source": [
-    "Probamos"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "id": "2735d1a1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pdf_path=\"./DOCS/Decreto-Supremo-N_-018-2019-JUS.pdf\" "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "id": "6ede8122",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pdf_chunks = pdfachunk(pdf_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "id": "8f33af13",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "split_docs = split_chunks(pdf_chunks)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "id": "b0fb3761",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "texts = [doc.page_content for doc in split_docs]"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "85f2ee3f",
-   "metadata": {},
-   "source": [
-    "Definimos nuestro modelo de embbending"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5eb18c36",
-   "metadata": {},
-   "source": [
-    "NECESITAMOS DATASETS Y EINOPS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "74262eaa",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%pip install datasets einops"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "id": "93bbbcde",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. Falling back to regular HTTP download. For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`\n"
-     ]
-    }
-   ],
-   "source": [
-    "model = SentenceTransformer(\"all-MiniLM-L6-v2\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "f9f4d5bd",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<class 'transformers_modules.nvidia.NV-Embed-v2.c50d55f43bde7e6a18e0eaa15a62fd63a930f1a1.modeling_nvembed.NVEmbedModel'>\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(type(model))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 29,
-   "id": "e594d6af",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Batches: 100%|██████████| 12/12 [00:03<00:00,  3.29it/s]\n"
-     ]
-    }
-   ],
-   "source": [
-    "embeddings = generaremben(model, texts)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "id": "beba3991",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "C:\\Users\\adm\\AppData\\Local\\Temp\\ipykernel_26272\\752761030.py:5: DeprecationWarning: `recreate_collection` method is deprecated and will be removed in the future. Use `collection_exists` to check collection existence and `create_collection` instead.\n",
-      "  client.recreate_collection(\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Insertados 181 vectores en Qdrant.\n"
-     ]
-    }
-   ],
-   "source": [
-    "insertarenqdra(embeddings, texts, collection_name=\"jus_decreto_018\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2c241dc6",
-   "metadata": {},
-   "source": [
-    "Funcion para consultar con qdrant"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 31,
-   "id": "86beaf73",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from qdrant_client import QdrantClient\n",
-    "import numpy as np"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b81de586",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def query_qdrant(query, model, collection_name, top_k=5):\n",
-    "    # Generar embedding de la consulta\n",
-    "    query_embedding = model.encode([query])[0]\n",
-    "    \n",
-    "    # Conexión al cliente Qdrant\n",
-    "    client = QdrantClient(path=\"./qdrant_data\")\n",
-    "\n",
-    "    # Realizar búsqueda en la colección\n",
-    "    results = client.search(\n",
-    "        collection_name=collection_name,\n",
-    "        query_vector=query_embedding.tolist(),\n",
-    "        limit=top_k,  # Limitar a los primeros K resultados más similares\n",
-    "        with_payload=True  # Incluir el texto en los resultados\n",
-    "    )\n",
-    "\n",
-    "    return results"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 36,
-   "id": "9d448736",
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "RuntimeError",
-     "evalue": "Storage folder ./qdrant_data is already accessed by another instance of Qdrant client. If you require concurrent access, use Qdrant server instead.",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31merror\u001b[39m                                     Traceback (most recent call last)",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\portalocker\\portalocker.py:49\u001b[39m, in \u001b[36mlock\u001b[39m\u001b[34m(file_, flags)\u001b[39m\n\u001b[32m     48\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m49\u001b[39m     \u001b[43mwin32file\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFileEx\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos_fh\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m-\u001b[49m\u001b[32;43m0x10000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m__overlapped\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     50\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m pywintypes.error \u001b[38;5;28;01mas\u001b[39;00m exc_value:\n\u001b[32m     51\u001b[39m     \u001b[38;5;66;03m# error: (33, 'LockFileEx', 'The process cannot access the file\u001b[39;00m\n\u001b[32m     52\u001b[39m     \u001b[38;5;66;03m# because another process has locked a portion of the file.')\u001b[39;00m\n",
-      "\u001b[31merror\u001b[39m: (33, 'LockFileEx', 'El proceso no tiene acceso al archivo porque otro proceso tiene bloqueada una parte del archivo.')",
-      "\nThe above exception was the direct cause of the following exception:\n",
-      "\u001b[31mAlreadyLocked\u001b[39m                             Traceback (most recent call last)",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:133\u001b[39m, in \u001b[36mQdrantLocal._load\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    132\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m133\u001b[39m     \u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlock\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    134\u001b[39m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_flock_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    135\u001b[39m \u001b[43m        \u001b[49m\u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFlags\u001b[49m\u001b[43m.\u001b[49m\u001b[43mEXCLUSIVE\u001b[49m\u001b[43m \u001b[49m\u001b[43m|\u001b[49m\u001b[43m \u001b[49m\u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFlags\u001b[49m\u001b[43m.\u001b[49m\u001b[43mNON_BLOCKING\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    136\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    137\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m portalocker.exceptions.LockException:\n",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\portalocker\\portalocker.py:54\u001b[39m, in \u001b[36mlock\u001b[39m\u001b[34m(file_, flags)\u001b[39m\n\u001b[32m     53\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:\n\u001b[32m---> \u001b[39m\u001b[32m54\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m exceptions.AlreadyLocked(\n\u001b[32m     55\u001b[39m         exceptions.LockException.LOCK_FAILED,\n\u001b[32m     56\u001b[39m         exc_value.strerror,\n\u001b[32m     57\u001b[39m         fh=file_,\n\u001b[32m     58\u001b[39m     ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mexc_value\u001b[39;00m\n\u001b[32m     59\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m     60\u001b[39m     \u001b[38;5;66;03m# Q:  Are there exceptions/codes we should be dealing with\u001b[39;00m\n\u001b[32m     61\u001b[39m     \u001b[38;5;66;03m# here?\u001b[39;00m\n",
-      "\u001b[31mAlreadyLocked\u001b[39m: (1, 'El proceso no tiene acceso al archivo porque otro proceso tiene bloqueada una parte del archivo.')",
-      "\nDuring handling of the above exception, another exception occurred:\n",
-      "\u001b[31mRuntimeError\u001b[39m                              Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[36]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m      1\u001b[39m query = \u001b[33m\"\u001b[39m\u001b[33m¿Cuál es el propósito de la Ley 018-2019?\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m results = \u001b[43mquery_qdrant\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mjus_decreto_018\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 6\u001b[39m, in \u001b[36mquery_qdrant\u001b[39m\u001b[34m(query, model, collection_name, top_k)\u001b[39m\n\u001b[32m      3\u001b[39m query_embedding = model.encode([query])[\u001b[32m0\u001b[39m]\n\u001b[32m      5\u001b[39m \u001b[38;5;66;03m# Conexión al cliente Qdrant\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m client = \u001b[43mQdrantClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m./qdrant_data\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m      8\u001b[39m \u001b[38;5;66;03m# Realizar búsqueda en la colección\u001b[39;00m\n\u001b[32m      9\u001b[39m results = client.search(\n\u001b[32m     10\u001b[39m     collection_name=collection_name,\n\u001b[32m     11\u001b[39m     query_vector=query_embedding.tolist(),\n\u001b[32m     12\u001b[39m     limit=top_k,  \u001b[38;5;66;03m# Limitar a los primeros K resultados más similares\u001b[39;00m\n\u001b[32m     13\u001b[39m     with_payload=\u001b[38;5;28;01mTrue\u001b[39;00m  \u001b[38;5;66;03m# Incluir el texto en los resultados\u001b[39;00m\n\u001b[32m     14\u001b[39m )\n",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\qdrant_client.py:133\u001b[39m, in \u001b[36mQdrantClient.__init__\u001b[39m\u001b[34m(self, location, url, port, grpc_port, prefer_grpc, https, api_key, prefix, timeout, host, path, force_disable_check_same_thread, grpc_options, auth_token_provider, cloud_inference, local_inference_batch_size, check_compatibility, **kwargs)\u001b[39m\n\u001b[32m    128\u001b[39m     \u001b[38;5;28mself\u001b[39m._client = QdrantLocal(\n\u001b[32m    129\u001b[39m         location=location,\n\u001b[32m    130\u001b[39m         force_disable_check_same_thread=force_disable_check_same_thread,\n\u001b[32m    131\u001b[39m     )\n\u001b[32m    132\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m133\u001b[39m     \u001b[38;5;28mself\u001b[39m._client = \u001b[43mQdrantLocal\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    134\u001b[39m \u001b[43m        \u001b[49m\u001b[43mlocation\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    135\u001b[39m \u001b[43m        \u001b[49m\u001b[43mforce_disable_check_same_thread\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_disable_check_same_thread\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    136\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    137\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m    138\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m location \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:66\u001b[39m, in \u001b[36mQdrantLocal.__init__\u001b[39m\u001b[34m(self, location, force_disable_check_same_thread)\u001b[39m\n\u001b[32m     64\u001b[39m \u001b[38;5;28mself\u001b[39m.aliases: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = {}\n\u001b[32m     65\u001b[39m \u001b[38;5;28mself\u001b[39m._flock_file: Optional[TextIOWrapper] = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m66\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_load\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m     67\u001b[39m \u001b[38;5;28mself\u001b[39m._closed: \u001b[38;5;28mbool\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m\n",
-      "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:138\u001b[39m, in \u001b[36mQdrantLocal._load\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m    133\u001b[39m     portalocker.lock(\n\u001b[32m    134\u001b[39m         \u001b[38;5;28mself\u001b[39m._flock_file,\n\u001b[32m    135\u001b[39m         portalocker.LockFlags.EXCLUSIVE | portalocker.LockFlags.NON_BLOCKING,\n\u001b[32m    136\u001b[39m     )\n\u001b[32m    137\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m portalocker.exceptions.LockException:\n\u001b[32m--> \u001b[39m\u001b[32m138\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m    139\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mStorage folder \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.location\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is already accessed by another instance of Qdrant client.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    140\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m If you require concurrent access, use Qdrant server instead.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    141\u001b[39m     )\n",
-      "\u001b[31mRuntimeError\u001b[39m: Storage folder ./qdrant_data is already accessed by another instance of Qdrant client. If you require concurrent access, use Qdrant server instead."
-     ]
-    }
-   ],
-   "source": [
-    "query = \"¿Cuál es el propósito de la Ley 018-2019?\"\n",
-    "results = query_qdrant(query, model,\"jus_decreto_018\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "61d76427",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": ".venv",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}