{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/arda/Desktop/A.I./Projects/FinanceChatbot'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import os\n",
    "import warnings\n",
    "from dotenv import load_dotenv\n",
    "from langchain.document_loaders import PyPDFLoader, DirectoryLoader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_qdrant import QdrantVectorStore\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "\n",
    "# Ignore all warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Environment variables loaded successfully.\n"
     ]
    }
   ],
   "source": [
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "\n",
    "# Check if .env file exists and API keys are loaded\n",
    "if not os.path.exists('.env'):\n",
    "    print(\"Warning: .env file not found!\")\n",
    "elif not os.getenv(\"QDRANT_API_KEY\") or not os.getenv(\"QDRANT_URL\"):\n",
    "    print(\"Warning: QDRANT_API_KEY or QDRANT_URL not found in .env file!\")\n",
    "else:\n",
    "    print(\"Environment variables loaded successfully.\")\n",
    "\n",
    "# Settings\n",
    "QDRANT_API_KEY = os.getenv(\"QDRANT_API_KEY\")\n",
    "QDRANT_URL = os.getenv(\"QDRANT_URL\")\n",
    "COLLECTION_NAME = \"finance-chatbot\"\n",
    "DATA_DIR = \"Data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique PDF files loaded: 3\n",
      "Loaded files:\n",
      "File 1: Data/Basics.pdf\n",
      "File 2: Data/Financialterms.pdf\n",
      "File 3: Data/Statementanalysis.pdf\n",
      "Success: All 3 PDFs (Basics.pdf, Statementanalysis.pdf, Financialterms.pdf) have been loaded.\n",
      "Total number of pages loaded: 547\n"
     ]
    }
   ],
   "source": [
    "# Load and extract data from PDFs\n",
    "def load_pdf_file(data_dir):\n",
    "    loader = DirectoryLoader(\n",
    "        data_dir,\n",
    "        glob=\"*.pdf\",\n",
    "        loader_cls=PyPDFLoader\n",
    "    )\n",
    "    documents = loader.load()\n",
    "    return documents\n",
    "\n",
    "extracted_data = load_pdf_file(DATA_DIR)\n",
    "\n",
    "# Verify the number of loaded PDFs by checking unique file sources\n",
    "unique_files = set(doc.metadata.get('source', 'Unknown') for doc in extracted_data)\n",
    "print(f\"Number of unique PDF files loaded: {len(unique_files)}\")\n",
    "print(\"Loaded files:\")\n",
    "for i, file in enumerate(unique_files, 1):\n",
    "    print(f\"File {i}: {file}\")\n",
    "\n",
    "# Check if the expected number of PDFs (3) were loaded\n",
    "if len(unique_files) == 3:\n",
    "    print(\"Success: All 3 PDFs (Basics.pdf, Statementanalysis.pdf, Financialterms.pdf) have been loaded.\")\n",
    "else:\n",
    "    print(f\"Warning: Expected 3 PDFs, but {len(unique_files)} unique files were loaded. Check the Data directory.\")\n",
    "\n",
    "# Additional info: Total number of pages (documents)\n",
    "print(f\"Total number of pages loaded: {len(extracted_data)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of Text Chunks: 2756\n"
     ]
    }
   ],
   "source": [
    "# Split the data into text chunks\n",
    "def text_split(extracted_data):\n",
    "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=20)\n",
    "    text_chunks = text_splitter.split_documents(extracted_data)\n",
    "    return text_chunks\n",
    "\n",
    "text_chunks = text_split(extracted_data)\n",
    "print(\"Length of Text Chunks:\", len(text_chunks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Dimension: 384\n"
     ]
    }
   ],
   "source": [
    "# Download embeddings from Hugging Face\n",
    "embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2')\n",
    "\n",
    "# Verify embedding dimension\n",
    "query_result = embeddings.embed_query(\"Hello world\")\n",
    "print(\"Embedding Dimension:\", len(query_result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Qdrant client and create/upload to collection\n",
    "try:\n",
    "    qdrant = QdrantVectorStore.from_documents(\n",
    "        documents=text_chunks,\n",
    "        embedding=embeddings,\n",
    "        url=QDRANT_URL,\n",
    "        api_key=QDRANT_API_KEY,\n",
    "        collection_name=COLLECTION_NAME\n",
    "    )\n",
    "    print(\"Qdrant collection created and populated successfully.\")\n",
    "except Exception as e:\n",
    "    print(f\"Error creating Qdrant collection: {e}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "finance_chatbot",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}