uodaded the test notebook

notebooks/1_test_pdf_reader.ipynb

@@ -15,6 +15,14 @@
    "execution_count": 2,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/sawale/Documents/learning/money_rag/.venv/lib/python3.12/site-packages/google/cloud/aiplatform/models.py:52: FutureWarning: Support for google-cloud-storage < 3.0.0 will be removed in a future version of google-cloud-aiplatform. Please upgrade to google-cloud-storage >= 3.0.0.\n",
+      "  from google.cloud.aiplatform.utils import gcs_utils\n"
+     ]
+    },
     {
      "data": {
       "text/plain": [
@@ -32,6 +40,22 @@
     "from langchain.chat_models import init_chat_model\n",
     "from langchain_core.prompts import ChatPromptTemplate\n",
     "from langchain_core.output_parsers import JsonOutputParser\n",
+    "import uuid\n",
+    "\n",
+    "from langchain_qdrant import QdrantVectorStore\n",
+    "from qdrant_client import QdrantClient\n",
+    "from qdrant_client.http.models import Distance, VectorParams\n",
+    "from langchain_google_vertexai import VertexAIEmbeddings\n",
+    "\n",
+    "from typing import TypedDict\n",
+    "from dataclasses import dataclass\n",
+    "from langgraph.runtime import get_runtime\n",
+    "from langgraph.checkpoint.memory import InMemorySaver\n",
+    "from langchain_community.utilities import SQLDatabase\n",
+    "from langchain_core.tools import tool\n",
+    "from langchain.agents import create_agent\n",
+    "\n",
+    "\n",
     "\n",
     "from dotenv import load_dotenv\n",
     "load_dotenv()"
@@ -39,16 +63,57 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
-    "def ingest_csv(file_path, llm, db_path=\"money_rag.db\"):\n",
+    "import os \n",
+    "import json\n",
+    "import asyncio\n",
+    "from langchain_community.tools import DuckDuckGoSearchRun\n",
+    "\n",
+    "# Initialize search\n",
+    "search_tool = DuckDuckGoSearchRun()\n",
+    "MERCHANT_CACHE_FILE = \"merchant_cache.json\"\n",
+    "\n",
+    "# Helper to load/save cache\n",
+    "def _load_cache():\n",
+    "    if os.path.exists(MERCHANT_CACHE_FILE):\n",
+    "        try:\n",
+    "            with open(MERCHANT_CACHE_FILE, 'r') as f:\n",
+    "                return json.load(f)\n",
+    "        except:\n",
+    "            return {}\n",
+    "    return {}\n",
+    "\n",
+    "def _save_cache(cache):\n",
+    "    with open(MERCHANT_CACHE_FILE, 'w') as f:\n",
+    "        json.dump(cache, f)\n",
+    "\n",
+    "async def get_merchant_category_async(description, cache, sem):\n",
+    "    \"\"\"Async search with semaphore and shared cache\"\"\"\n",
+    "    if description in cache:\n",
+    "        return cache[description]\n",
+    "    \n",
+    "    async with sem:\n",
+    "        try:\n",
+    "            # Random small sleep to jitter requests slightly\n",
+    "            await asyncio.sleep(0.05) \n",
+    "            print(f\"   🔍 Web searching for: {description}...\")\n",
+    "            # Run the search asynchronously\n",
+    "            result = await search_tool.ainvoke(f\"What type of business / store is '{description}'? Whats is location\")\n",
+    "            return result\n",
+    "        except Exception as e:\n",
+    "            print(f\"   ⚠️ Search failed for {description}: {e}\")\n",
+    "            return \"Unknown\"\n",
+    "\n",
+    "async def ingest_csv(file_path, llm, db_path=\"money_rag.db\"):\n",
+    "    print(f\"📂 Processing {file_path}...\")\n",
     "    df = pd.read_csv(file_path)\n",
     "    headers = df.columns.tolist()\n",
-    "    # Taking 10 rows gives the LLM a better chance to see both a purchase and a payment\n",
     "    sample_data = df.head(10).to_json() \n",
     "\n",
+    "    # 1. LLM Mapping (Sync is fine here)\n",
     "    prompt = ChatPromptTemplate.from_template(\"\"\"\n",
     "    Act as a financial data parser. Analyze this CSV data:\n",
     "    Filename: {filename}\n",
@@ -57,7 +122,7 @@
     "\n",
     "    TASK:\n",
     "    1. Map the CSV columns to standard fields: date, description, amount, and category.\n",
-    "    2. Determine the 'sign_convention' by looking at the filename and the data:\n",
+    "    2. Determine the 'sign_convention' for spending.\n",
     "       \n",
     "       RULES:\n",
     "       - If the filename suggests 'Discover' credit card, spending are usually POSITIVE.\n",
@@ -67,7 +132,6 @@
     "          - Look at the sample data for known merchants or spending patterns.\n",
     "          - If spending (like a restaurant or store) is NEGATIVE (e.g., -25.00), the convention is 'spending_is_negative'.\n",
     "          - If spending is POSITIVE (e.g., 25.00), the convention is 'spending_is_positive'.\n",
-    "                                              \n",
     "\n",
     "    OUTPUT FORMAT (JSON ONLY):\n",
     "    {{\n",
@@ -82,53 +146,216 @@
     "    chain = prompt | llm | JsonOutputParser()\n",
     "    mapping = chain.invoke({\"headers\": headers, \"sample\": sample_data, \"filename\": file_path})\n",
     "\n",
-    "    # Standardizing Data\n",
+    "    # 2. Standardization\n",
     "    standard_df = pd.DataFrame()\n",
+    "    standard_df['id'] = [str(uuid.uuid4()) for _ in range(len(df))]\n",
     "    standard_df['transaction_date'] = pd.to_datetime(df[mapping['date_col']])\n",
     "    standard_df['description'] = df[mapping['desc_col']]\n",
     "    \n",
-    "    # --- The Normalization Logic ---\n",
-    "    # Goal: All spending (outflow) = POSITIVE, All payments (inflow) = NEGATIVE\n",
+    "    # Normalization Logic\n",
     "    raw_amounts = pd.to_numeric(df[mapping['amount_col']])\n",
-    "    \n",
     "    if mapping['sign_convention'] == \"spending_is_negative\":\n",
-    "        # If the bank shows spending as -100 and payments as +100, \n",
-    "        # we flip everything so spending is +100 and payments are -100.\n",
     "        standard_df['amount'] = raw_amounts * -1\n",
     "    else:\n",
-    "        # If the bank already shows spending as +100 and payments as -100, keep it.\n",
     "        standard_df['amount'] = raw_amounts\n",
     "    \n",
     "    standard_df['category'] = df[mapping['category_col']] if mapping.get('category_col') else 'Uncategorized'\n",
     "    standard_df['source_file'] = file_path.split(\"/\")[-1]\n",
     "\n",
-    "    # Save to DB\n",
+    "    # 3. --- Async Enrichment Step ---\n",
+    "    print(\"   ✨ Enriching descriptions (Async)...\")\n",
+    "    unique_descriptions = standard_df['description'].unique()\n",
+    "    \n",
+    "    # Load cache once\n",
+    "    cache = _load_cache()\n",
+    "    \n",
+    "    # Create a semaphore to limit concurrent web searches (e.g. 5)\n",
+    "    sem = asyncio.Semaphore(5)\n",
+    "    \n",
+    "    # Create tasks for all descriptions\n",
+    "    # Note: We process ALL descriptions now effectively\n",
+    "    tasks = []\n",
+    "    for desc in unique_descriptions:\n",
+    "        tasks.append(get_merchant_category_async(desc, cache, sem))\n",
+    "        \n",
+    "    # Run in parallel\n",
+    "    results = await asyncio.gather(*tasks)\n",
+    "    \n",
+    "    # Update cache object with new results\n",
+    "    for desc, res in zip(unique_descriptions, results):\n",
+    "        cache[desc] = res\n",
+    "        \n",
+    "    # Save cache back to disk\n",
+    "    _save_cache(cache)\n",
+    "        \n",
+    "    # Map back\n",
+    "    desc_map = dict(zip(unique_descriptions, results))\n",
+    "    standard_df['enriched_info'] = standard_df['description'].map(desc_map).fillna(\"\")\n",
+    "\n",
+    "    # 4. Save to DB\n",
     "    conn = sqlite3.connect(db_path)\n",
     "    standard_df.to_sql(\"transactions\", conn, if_exists=\"append\", index=False)\n",
     "    conn.close()\n",
     "    \n",
-    "    print(f\"✅ Ingested {file_path}. Logic: {mapping['sign_convention']}\")"
+    "    print(f\"✅ Ingested {len(standard_df)} rows from {file_path.split('/')[-1]}. Logic: {mapping['sign_convention']}\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/sawale/Documents/learning/money_rag/.venv/lib/python3.12/site-packages/google/cloud/aiplatform/models.py:52: FutureWarning: Support for google-cloud-storage < 3.0.0 will be removed in a future version of google-cloud-aiplatform. Please upgrade to google-cloud-storage >= 3.0.0.\n",
-      "  from google.cloud.aiplatform.utils import gcs_utils\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Ingested /Users/sawale/Documents/learning/money_rag/demo_data/Discover-AllAvailable-20260110.csv. Logic: spending_is_positive\n",
-      "✅ Ingested /Users/sawale/Documents/learning/money_rag/demo_data/Chase5282_Activity20240110_20260110_20260111.CSV. Logic: spending_is_negative\n"
+      "📂 Processing /Users/sawale/Documents/learning/money_rag/demo_data/Discover-AllAvailable-20260110.csv...\n",
+      "   ✨ Enriching descriptions (Async)...\n",
+      "   🔍 Web searching for: BACK MARKET BROOKLYN NY...\n",
+      "   🔍 Web searching for: TEMU.COM 8884958368 DE...\n",
+      "   🔍 Web searching for: WALMART STORE 00332 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: $100 STATEMENT CREDIT W 1ST PU...\n",
+      "   🔍 Web searching for: PY *KUNG-FU TEA AL HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: MADISON MONTGOMERY AL...\n",
+      "   🔍 Web searching for: INTERNET PAYMENT - THANK YOU...\n",
+      "   🔍 Web searching for: GRUBHUB - UNIVERSITY OF HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: MINT MOBILE 800-683-7392 CA...\n",
+      "   🔍 Web searching for: POPEYES 2577 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: 88 BUFFET HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: VIET HUONG VIETNAMESE RE HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: CASHBACK BONUS REDEMPTION PYMT/STMT CRDT...\n",
+      "   🔍 Web searching for: SPO*THECURRYMODERNINDIAN HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: H&M 0273HUNTSVILLE HUNTSVILLE ALUS0273001241222182740...\n",
+      "   🔍 Web searching for: INDIAN BAZAAR HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: HANDELS HOMEMADE HUNTSVI HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: UAH COLLEGE 256-824-6170 AL...\n",
+      "   🔍 Web searching for: UAH COLLEGE FSF 800-346-9252 MA...\n",
+      "   🔍 Web searching for: CHIPOTLE 1687 NASHVILLE TN...\n",
+      "   🔍 Web searching for: TST*PIE TOWN TACOS - F NASHVILLE TN00153526022200965677AA...\n",
+      "   🔍 Web searching for: INDIAN BAZAAR HUNTSVILLE ALGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: INDIA MART HUNTSVILLE ALGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: PAYPAL *KEVDUDE1186 KEV 888-221-1161 CA...\n",
+      "   🔍 Web searching for: LYFT *RIDE WED 10AM 8552800278 CA...\n",
+      "   🔍 Web searching for: SKECHERS USA INC 1069 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: STORE HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: LYFT *RIDE WED 3PM 8552800278 CA...\n",
+      "   🔍 Web searching for: SQ *TAQUERIA LAS ADELI HUNTSVILLE AL0002305843021411201895...\n",
+      "   🔍 Web searching for: UAH HUNTSVILLE DUNKIN HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: WALMART.COM 800-925-6278 AR...\n",
+      "   🔍 Web searching for: WALMART.COM 8009256278 BENTONVILLE AR...\n",
+      "   🔍 Web searching for: TOUS LES JOURS - HUNTSVI HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: MARSHALLS #422 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: ROSS STORE #2436 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: SPRINTAX NR TAX 8882038900 NY...\n",
+      "   🔍 Web searching for: USPS PO 0142460804 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: CHIPOTLE 1796 HUNTSVILLE ALGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: TST*POURHOUSE HUNTSVILLE AL00031984024314246667AA...\n",
+      "   🔍 Web searching for: TST*WOKS UP HUNTSVILLE AL00075396024313993332AA...\n",
+      "   🔍 Web searching for: SPIRIT AIRLINES 8014012222 FL...\n",
+      "   🔍 Web searching for: CHIPOTLE 1796 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: UAH BURSARS OFFICE HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: STARS AND STRIKES - HUNT HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: ROSS STORES #620 HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: TST*KAMADO RAMEN - MID HUNTSVILLE AL00006963025030352515AA...\n",
+      "   🔍 Web searching for: SQ *MOM'SCLAYCO HARVEST AL0002305843022068424398...\n",
+      "   🔍 Web searching for: DOLLARTREE HUNTSVILLE AL...\n",
+      "   🔍 Web searching for: SLIM & HUSKIES NASHVILLE TN...\n",
+      "   🔍 Web searching for: CHIPOTLE 1392 SANTA MONICA CA...\n",
+      "   🔍 Web searching for: DOLLAR TREE LAS VEGAS NV...\n",
+      "   🔍 Web searching for: LYFT *RIDE TUE 12AM 8552800278 CA...\n",
+      "   🔍 Web searching for: SQ *SHIKU GCM LOS ANGELES CA0001152921515467218869...\n",
+      "   🔍 Web searching for: SQ *SHIKU GCM LOS ANGELES CA0001152921515467211997...\n",
+      "   🔍 Web searching for: WALMART STORE 05686 BURBANK CA...\n",
+      "   🔍 Web searching for: CAFE BELLA NEWPORT SAN DIEGO CAGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: CHIPOTLE 2883 NORTH LAS VEGNVGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: SHELL10006319007 HESPERIA CAGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: PANDA EXPRESS #1964 LAS VEGAS NV...\n",
+      "   🔍 Web searching for: DENNY'S #0141 QR LAS VEGAS NVGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: LAS VEGAS SOUVENIRS AND LAS VEGAS NV...\n",
+      "   🔍 Web searching for: CTLP*FIRST CLASS VENDI BELLGARDENS CA...\n",
+      "   🔍 Web searching for: SHELL12874333011 FRANKLIN TN...\n",
+      "   🔍 Web searching for: AMARAVATI INDIAN CUISINE BRENTWOOD TNGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: CENTRAL MARKET NASHVILLE TN...\n",
+      "   🔍 Web searching for: TST*PRINCES HOT CHICKE NASHVILLE TN00104605025320544723AA...\n",
+      "   🔍 Web searching for: TST*PRINCES HOT CHICKE NASHVILLE TN00104605025321087148AA...\n",
+      "   🔍 Web searching for: WALMART STORE 05616 NASHVILLE TN...\n",
+      "   🔍 Web searching for: PY *KUNG-FU TEA AL HUNTSVILLE ALGOOGLE PAY ENDING IN 8984...\n",
+      "   🔍 Web searching for: 2LEVY R&C CHATTANOOGA TNGOOGLE PAY ENDING IN 8984...\n",
+      "✅ Ingested 124 rows from Discover-AllAvailable-20260110.csv. Logic: spending_is_positive\n",
+      "📂 Processing /Users/sawale/Documents/learning/money_rag/demo_data/Chase5282_Activity20240110_20260110_20260111.CSV...\n",
+      "   ✨ Enriching descriptions (Async)...\n",
+      "   🔍 Web searching for: TOUS LES JOURS - HUNTSVIL...\n",
+      "   🔍 Web searching for: Payment Thank You-Mobile...\n",
+      "   ��� Web searching for: INDIAN BAZAAR...\n",
+      "   🔍 Web searching for: TST*BLUE OAK BBQ-HUNTSVI...\n",
+      "   🔍 Web searching for: AMC 4112 VAL BEND 18...\n",
+      "   🔍 Web searching for: HANDELS HOMEMADE JONES V...\n",
+      "   🔍 Web searching for: PAYYOURSELFBACK CREDIT...\n",
+      "   🔍 Web searching for: TST* HYDERABAD HOUSE...\n",
+      "   🔍 Web searching for: PATEL BROTHERS NASHVILLE...\n",
+      "   🔍 Web searching for: CITY OF HUNTSVILLE...\n",
+      "   🔍 Web searching for: WM SUPERCENTER #332...\n",
+      "   🔍 Web searching for: WAL-MART #0332...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*OS1RI3LN3...\n",
+      "   🔍 Web searching for: TST* HATTIE B'S HUNTSVILL...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BI23Z6JR0...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BI9IW9OS2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BI0296OJ2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BB71A2881...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BB3FU2UQ2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BI03P1OX2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BB92U9QK2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BB9TA14Q0...\n",
+      "   🔍 Web searching for: 88 BUFFET...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*BB0DC71B1...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*B20NN4ID0...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*B273C1WY2...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*B27IN41E1...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*B250Z60P1...\n",
+      "   🔍 Web searching for: BEST BUY      00005140...\n",
+      "   🔍 Web searching for: DAVES HOT CHICKEN 1282...\n",
+      "   🔍 Web searching for: SQ *VIETCUISINE LLC...\n",
+      "   🔍 Web searching for: CHICK-FIL-A #00579...\n",
+      "   🔍 Web searching for: COSTCO WHSE #0356...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*NK4AM43Q2...\n",
+      "   🔍 Web searching for: HUNTSVILLE FLV...\n",
+      "   🔍 Web searching for: AMAZON MKTPL*NM1H055K0...\n",
+      "   🔍 Web searching for: MAPCO EXPRESS #3403...\n",
+      "   🔍 Web searching for: DUNKIN #346212 Q35...\n",
+      "   🔍 Web searching for: CENTRAL MARKET...\n",
+      "   🔍 Web searching for: TARA INTERNATIONAL MARKET...\n",
+      "   🔍 Web searching for: BOTAN MARKET INC...\n",
+      "   🔍 Web searching for: AMARAVATI INDIAN CUISINE...\n",
+      "   🔍 Web searching for: GRUBHUB - UNIVERSITY OF A...\n",
+      "   🔍 Web searching for: BURGER KING #4959...\n",
+      "   🔍 Web searching for: PANDA EXPRESS #3013...\n",
+      "   🔍 Web searching for: MCDONALD'S F2431...\n",
+      "   🔍 Web searching for: ENDZONE COLLECTIBLES...\n",
+      "   🔍 Web searching for: ZIMMAD EVE* ZIMMAD JOI...\n",
+      "   🔍 Web searching for: SQ *SPILL COFFEE AND CREA...\n",
+      "   🔍 Web searching for: 10267 CAVA WHITESBURG...\n",
+      "   🔍 Web searching for: SPO*DRAGONSFORGECAFE...\n",
+      "   🔍 Web searching for: UAH BURSARS OFFICE...\n",
+      "   🔍 Web searching for: MARATHON PETRO42804...\n",
+      "   🔍 Web searching for: TST*NOTHING BUT NOODLES...\n",
+      "   🔍 Web searching for: VEDA INDIAN CUISINE...\n",
+      "   🔍 Web searching for: DOLLARTREE...\n",
+      "   🔍 Web searching for: TARGET        00013466...\n",
+      "   🔍 Web searching for: POPEYES 2577...\n",
+      "   🔍 Web searching for: DEORALI GROCERY...\n",
+      "   🔍 Web searching for: HELLO ATLANTA #33...\n",
+      "   🔍 Web searching for: SKY VIEW ATLANTA...\n",
+      "   🔍 Web searching for: STARBUCKS 25111...\n",
+      "   🔍 Web searching for: BP#8998205AM/PM WADE GRE...\n",
+      "   🔍 Web searching for: Waffle House 0857...\n",
+      "   🔍 Web searching for: CINEMARK 1131 BOXCON...\n",
+      "   🔍 Web searching for: CINEMARK 1131 RSTBAR...\n",
+      "   🔍 Web searching for: HOMEGOODS # 0568...\n",
+      "   🔍 Web searching for: ASIAN MARKET...\n",
+      "   🔍 Web searching for: PANDA EXPRESS #2005...\n",
+      "   🔍 Web searching for: STARS AND STRIKES - HUNTS...\n",
+      "   🔍 Web searching for: WAL-MART #332...\n",
+      "✅ Ingested 126 rows from Chase5282_Activity20240110_20260110_20260111.CSV. Logic: spending_is_negative\n"
      ]
     }
    ],
@@ -137,21 +364,23 @@
     "path2 = \"/Users/sawale/Documents/learning/money_rag/demo_data/Chase5282_Activity20240110_20260110_20260111.CSV\"\n",
     "\n",
     "# Initialize the Gemini model via Vertex AI\n",
-    "llm = init_chat_model(\n",
+    "vertex_llm = init_chat_model(\n",
     "    \"gemini-2.5-flash\", \n",
     "    model_provider=\"google_vertexai\",\n",
     "    project='gen-lang-client-0311515393',\n",
     "    location='us-central1',\n",
     ")\n",
     "\n",
-    "\n",
-    "ingest_csv(path1, llm)\n",
-    "ingest_csv(path2, llm)\n"
+    "# Run async functions in Jupyter\n",
+    "# We run them sequentially here to avoid file lock issues with SQLite if both try to write at once\n",
+    "# (Though SQLite handles concurrency, keeping ingestion strictly ordered is safer for the demo)\n",
+    "await ingest_csv(path1, vertex_llm)\n",
+    "await ingest_csv(path2, vertex_llm)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -175,53 +404,65 @@
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
+       "      <th>id</th>\n",
        "      <th>transaction_date</th>\n",
        "      <th>description</th>\n",
        "      <th>amount</th>\n",
        "      <th>category</th>\n",
        "      <th>source_file</th>\n",
+       "      <th>enriched_info</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
+       "      <td>8ea03f61-dd51-45f9-a633-bd363154a424</td>\n",
        "      <td>2024-10-17 00:00:00</td>\n",
        "      <td>BACK MARKET BROOKLYN NY</td>\n",
        "      <td>231.19</td>\n",
        "      <td>Merchandise</td>\n",
        "      <td>Discover-AllAvailable-20260110.csv</td>\n",
+       "      <td>Online Retailer in New York, NY . See BBB rati...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
+       "      <td>02107b64-e0bd-4a7f-b5cc-674b7767a50f</td>\n",
        "      <td>2024-10-18 00:00:00</td>\n",
        "      <td>TEMU.COM 8884958368 DE</td>\n",
        "      <td>16.51</td>\n",
        "      <td>Merchandise</td>\n",
        "      <td>Discover-AllAvailable-20260110.csv</td>\n",
+       "      <td>Temu ' s business model has allowed it to beco...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
+       "      <td>a5a7cc9f-46ab-4913-a695-290b75f590a9</td>\n",
        "      <td>2024-10-18 00:00:00</td>\n",
        "      <td>WALMART STORE 00332 HUNTSVILLE AL</td>\n",
        "      <td>146.73</td>\n",
        "      <td>Merchandise</td>\n",
        "      <td>Discover-AllAvailable-20260110.csv</td>\n",
+       "      <td>Walmart Inc. is an American multinational reta...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
+       "      <td>332c4fac-7760-4c63-992b-fd24fcdd4eee</td>\n",
        "      <td>2024-10-18 00:00:00</td>\n",
        "      <td>$100 STATEMENT CREDIT W 1ST PU</td>\n",
        "      <td>-100.00</td>\n",
        "      <td>Awards and Rebate Credits</td>\n",
        "      <td>Discover-AllAvailable-20260110.csv</td>\n",
+       "      <td>All U.S. Bank credit cards offer contactless c...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
+       "      <td>ef01b0e5-5a29-4c08-ac16-fff028cc23e6</td>\n",
        "      <td>2024-11-02 00:00:00</td>\n",
        "      <td>PY *KUNG-FU TEA AL HUNTSVILLE AL</td>\n",
        "      <td>8.09</td>\n",
        "      <td>Restaurants</td>\n",
        "      <td>Discover-AllAvailable-20260110.csv</td>\n",
+       "      <td>Jan 22, 2021 · Best part to me--besides the ro...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -230,102 +471,127 @@
        "      <td>...</td>\n",
        "      <td>...</td>\n",
        "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>245</th>\n",
+       "      <td>8c5d2425-5ddd-4532-8279-d6b3e733ec01</td>\n",
        "      <td>2025-06-18 00:00:00</td>\n",
        "      <td>PANDA EXPRESS #2005</td>\n",
        "      <td>52.87</td>\n",
        "      <td>Food &amp; Drink</td>\n",
        "      <td>Chase5282_Activity20240110_20260110_20260111.CSV</td>\n",
+       "      <td>This is a list of notable current and former f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>246</th>\n",
+       "      <td>02578b56-f0a1-490d-8cf1-dd061a26ebf2</td>\n",
        "      <td>2025-06-14 00:00:00</td>\n",
        "      <td>Payment Thank You-Mobile</td>\n",
        "      <td>-62.07</td>\n",
        "      <td>None</td>\n",
        "      <td>Chase5282_Activity20240110_20260110_20260111.CSV</td>\n",
+       "      <td>Locate a mobile phone | GSM locator of mobile ...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>247</th>\n",
+       "      <td>e0acc481-3baf-458b-9cdf-75bc1524a5f5</td>\n",
        "      <td>2025-06-12 00:00:00</td>\n",
        "      <td>STARS AND STRIKES - HUNTS</td>\n",
        "      <td>21.80</td>\n",
        "      <td>Entertainment</td>\n",
        "      <td>Chase5282_Activity20240110_20260110_20260111.CSV</td>\n",
+       "      <td>At our Huntsville , AL location , we pride our...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>248</th>\n",
+       "      <td>dfc97eee-5207-464a-b6e1-a49fd80df48c</td>\n",
        "      <td>2025-06-11 00:00:00</td>\n",
        "      <td>WAL-MART #332</td>\n",
        "      <td>4.47</td>\n",
        "      <td>Groceries</td>\n",
        "      <td>Chase5282_Activity20240110_20260110_20260111.CSV</td>\n",
+       "      <td>From toys and video games to fashionable cloth...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>249</th>\n",
+       "      <td>576a6f5a-d090-4e4c-84f1-c1cc3fe8809c</td>\n",
        "      <td>2025-06-11 00:00:00</td>\n",
        "      <td>WAL-MART #332</td>\n",
        "      <td>57.60</td>\n",
        "      <td>Groceries</td>\n",
        "      <td>Chase5282_Activity20240110_20260110_20260111.CSV</td>\n",
+       "      <td>From toys and video games to fashionable cloth...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>250 rows × 5 columns</p>\n",
+       "<p>250 rows × 7 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "        transaction_date                        description  amount  \\\n",
-       "0    2024-10-17 00:00:00            BACK MARKET BROOKLYN NY  231.19   \n",
-       "1    2024-10-18 00:00:00             TEMU.COM 8884958368 DE   16.51   \n",
-       "2    2024-10-18 00:00:00  WALMART STORE 00332 HUNTSVILLE AL  146.73   \n",
-       "3    2024-10-18 00:00:00     $100 STATEMENT CREDIT W 1ST PU -100.00   \n",
-       "4    2024-11-02 00:00:00   PY *KUNG-FU TEA AL HUNTSVILLE AL    8.09   \n",
-       "..                   ...                                ...     ...   \n",
-       "245  2025-06-18 00:00:00                PANDA EXPRESS #2005   52.87   \n",
-       "246  2025-06-14 00:00:00           Payment Thank You-Mobile  -62.07   \n",
-       "247  2025-06-12 00:00:00          STARS AND STRIKES - HUNTS   21.80   \n",
-       "248  2025-06-11 00:00:00                      WAL-MART #332    4.47   \n",
-       "249  2025-06-11 00:00:00                      WAL-MART #332   57.60   \n",
+       "                                       id     transaction_date  \\\n",
+       "0    8ea03f61-dd51-45f9-a633-bd363154a424  2024-10-17 00:00:00   \n",
+       "1    02107b64-e0bd-4a7f-b5cc-674b7767a50f  2024-10-18 00:00:00   \n",
+       "2    a5a7cc9f-46ab-4913-a695-290b75f590a9  2024-10-18 00:00:00   \n",
+       "3    332c4fac-7760-4c63-992b-fd24fcdd4eee  2024-10-18 00:00:00   \n",
+       "4    ef01b0e5-5a29-4c08-ac16-fff028cc23e6  2024-11-02 00:00:00   \n",
+       "..                                    ...                  ...   \n",
+       "245  8c5d2425-5ddd-4532-8279-d6b3e733ec01  2025-06-18 00:00:00   \n",
+       "246  02578b56-f0a1-490d-8cf1-dd061a26ebf2  2025-06-14 00:00:00   \n",
+       "247  e0acc481-3baf-458b-9cdf-75bc1524a5f5  2025-06-12 00:00:00   \n",
+       "248  dfc97eee-5207-464a-b6e1-a49fd80df48c  2025-06-11 00:00:00   \n",
+       "249  576a6f5a-d090-4e4c-84f1-c1cc3fe8809c  2025-06-11 00:00:00   \n",
+       "\n",
+       "                           description  amount                   category  \\\n",
+       "0              BACK MARKET BROOKLYN NY  231.19                Merchandise   \n",
+       "1               TEMU.COM 8884958368 DE   16.51                Merchandise   \n",
+       "2    WALMART STORE 00332 HUNTSVILLE AL  146.73                Merchandise   \n",
+       "3       $100 STATEMENT CREDIT W 1ST PU -100.00  Awards and Rebate Credits   \n",
+       "4     PY *KUNG-FU TEA AL HUNTSVILLE AL    8.09                Restaurants   \n",
+       "..                                 ...     ...                        ...   \n",
+       "245                PANDA EXPRESS #2005   52.87               Food & Drink   \n",
+       "246           Payment Thank You-Mobile  -62.07                       None   \n",
+       "247          STARS AND STRIKES - HUNTS   21.80              Entertainment   \n",
+       "248                      WAL-MART #332    4.47                  Groceries   \n",
+       "249                      WAL-MART #332   57.60                  Groceries   \n",
        "\n",
-       "                      category  \\\n",
-       "0                  Merchandise   \n",
-       "1                  Merchandise   \n",
-       "2                  Merchandise   \n",
-       "3    Awards and Rebate Credits   \n",
-       "4                  Restaurants   \n",
-       "..                         ...   \n",
-       "245               Food & Drink   \n",
-       "246                       None   \n",
-       "247              Entertainment   \n",
-       "248                  Groceries   \n",
-       "249                  Groceries   \n",
+       "                                          source_file  \\\n",
+       "0                  Discover-AllAvailable-20260110.csv   \n",
+       "1                  Discover-AllAvailable-20260110.csv   \n",
+       "2                  Discover-AllAvailable-20260110.csv   \n",
+       "3                  Discover-AllAvailable-20260110.csv   \n",
+       "4                  Discover-AllAvailable-20260110.csv   \n",
+       "..                                                ...   \n",
+       "245  Chase5282_Activity20240110_20260110_20260111.CSV   \n",
+       "246  Chase5282_Activity20240110_20260110_20260111.CSV   \n",
+       "247  Chase5282_Activity20240110_20260110_20260111.CSV   \n",
+       "248  Chase5282_Activity20240110_20260110_20260111.CSV   \n",
+       "249  Chase5282_Activity20240110_20260110_20260111.CSV   \n",
        "\n",
-       "                                          source_file  \n",
-       "0                  Discover-AllAvailable-20260110.csv  \n",
-       "1                  Discover-AllAvailable-20260110.csv  \n",
-       "2                  Discover-AllAvailable-20260110.csv  \n",
-       "3                  Discover-AllAvailable-20260110.csv  \n",
-       "4                  Discover-AllAvailable-20260110.csv  \n",
-       "..                                                ...  \n",
-       "245  Chase5282_Activity20240110_20260110_20260111.CSV  \n",
-       "246  Chase5282_Activity20240110_20260110_20260111.CSV  \n",
-       "247  Chase5282_Activity20240110_20260110_20260111.CSV  \n",
-       "248  Chase5282_Activity20240110_20260110_20260111.CSV  \n",
-       "249  Chase5282_Activity20240110_20260110_20260111.CSV  \n",
+       "                                         enriched_info  \n",
+       "0    Online Retailer in New York, NY . See BBB rati...  \n",
+       "1    Temu ' s business model has allowed it to beco...  \n",
+       "2    Walmart Inc. is an American multinational reta...  \n",
+       "3    All U.S. Bank credit cards offer contactless c...  \n",
+       "4    Jan 22, 2021 · Best part to me--besides the ro...  \n",
+       "..                                                 ...  \n",
+       "245  This is a list of notable current and former f...  \n",
+       "246  Locate a mobile phone | GSM locator of mobile ...  \n",
+       "247  At our Huntsville , AL location , we pride our...  \n",
+       "248  From toys and video games to fashionable cloth...  \n",
+       "249  From toys and video games to fashionable cloth...  \n",
        "\n",
-       "[250 rows x 5 columns]"
+       "[250 rows x 7 columns]"
       ]
      },
-     "execution_count": 1,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# ...existing code...\n",
+    "\n",
     "import sqlite3\n",
     "import pandas as pd\n",
     "\n",
@@ -344,24 +610,306 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_view[\"amount\"].sum()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "np.float64(57.089999999999804)"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/sawale/Documents/learning/money_rag/.venv/lib/python3.12/site-packages/vertexai/_model_garden/_model_garden_models.py:278: UserWarning: This feature is deprecated as of June 24, 2025 and will be removed on June 24, 2026. For details, see https://cloud.google.com/vertex-ai/generative-ai/docs/deprecations/genai-vertexai-sdk.\n",
+      "  warning_logs.show_deprecation_warning()\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Synced 250 records to Qdrant.\n"
+     ]
     }
    ],
    "source": [
-    "df_view[\"amount\"].sum()"
+    "db = SQLDatabase.from_uri(\"sqlite:///money_rag.db\")\n",
+    "\n",
+    "embeddings = VertexAIEmbeddings(model_name=\"text-embedding-005\")\n",
+    "\n",
+    "\n",
+    "# Initialize Qdrant with disk persistence\n",
+    "# \"path\" creates a local directory to store the vectors\n",
+    "qdrant_client = QdrantClient(path=\"qdrant_db\") \n",
+    "COLLECTION_NAME = \"transactions\"\n",
+    "\n",
+    "def sync_to_qdrant(db_path: str):\n",
+    "    \"\"\"Sync the transactions table from SQLite to Qdrant vector store.\"\"\"\n",
+    "    # Load data from the database\n",
+    "    conn = sqlite3.connect(db_path)\n",
+    "    # Ensure we select the new column\n",
+    "    df = pd.read_sql_query(\"SELECT * FROM transactions\", conn)\n",
+    "    conn.close()\n",
+    "\n",
+    "    # Recreate collection to ensure clean state on re-ingestion\n",
+    "    if qdrant_client.collection_exists(COLLECTION_NAME):\n",
+    "        qdrant_client.delete_collection(COLLECTION_NAME)\n",
+    "        \n",
+    "    qdrant_client.create_collection(\n",
+    "        collection_name=COLLECTION_NAME,\n",
+    "        vectors_config=VectorParams(size=768, distance=Distance.COSINE),\n",
+    "    )\n",
+    "    \n",
+    "    vector_store = QdrantVectorStore(\n",
+    "        client=qdrant_client,\n",
+    "        collection_name=COLLECTION_NAME,\n",
+    "        embedding=embeddings,\n",
+    "    )\n",
+    "\n",
+    "    # Use description + enrichment as the main text for embedding\n",
+    "    texts = []\n",
+    "    for _, row in df.iterrows():\n",
+    "        # Combine original description with the permanent enriched info\n",
+    "        # If enriched_info is present, it looks like: \"MCDONALDS - Fast food chain...\"\n",
+    "        enriched = row.get('enriched_info', '')\n",
+    "        if enriched:\n",
+    "            texts.append(f\"{row['description']} - {enriched}\")\n",
+    "        else:\n",
+    "            texts.append(str(row['description']))\n",
+    "    \n",
+    "    # Store other fields as metadata for correlation\n",
+    "    metadatas = df[['id', 'amount', 'category', 'transaction_date']].to_dict('records')\n",
+    "    # Convert timestamps to string for metadata compatibility\n",
+    "    for m in metadatas:\n",
+    "        m['transaction_date'] = str(m['transaction_date'])\n",
+    "\n",
+    "    vector_store.add_texts(texts=texts, metadatas=metadatas)\n",
+    "    print(f\"✅ Synced {len(texts)} records to Qdrant at 'qdrant_db/'.\")\n",
+    "    return vector_store\n",
+    "\n",
+    "# Initialize store from DB\n",
+    "vector_store = sync_to_qdrant(\"money_rag.db\")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@dataclass\n",
+    "class RuntimeContext:\n",
+    "    db: SQLDatabase\n",
+    "    vector_store: QdrantVectorStore\n",
+    "\n",
+    "@tool\n",
+    "def execute_sql(query: str) -> str:\n",
+    "    \"\"\"Execute a SQLite command and return results.\"\"\"\n",
+    "    runtime = get_runtime(RuntimeContext)\n",
+    "    db = runtime.context.db\n",
+    "    try:\n",
+    "        return db.run(query)\n",
+    "    except Exception as e:\n",
+    "        return f\"Error: {e}\"\n",
+    "\n",
+    "\n",
+    "@tool\n",
+    "def semantic_search(query: str, topk: int = 5) -> str:\n",
+    "    \"\"\"\n",
+    "    Search for transactions semantically when exact category or description matches are unknown.\n",
+    "    Use this to find specific merchants or types of spending (e.g., 'streaming services' or 'fast food').\n",
+    "    Returns a list of matching transactions with their IDs and metadata.\n",
+    "    \"\"\"\n",
+    "    runtime = get_runtime(RuntimeContext)\n",
+    "    vs = runtime.context.vector_store\n",
+    "    results = vs.similarity_search(query, k=topk)\n",
+    "    \n",
+    "    output = []\n",
+    "    for doc in results:\n",
+    "        output.append(f\"Result: {doc.page_content} | Metadata: {doc.metadata}\")\n",
+    "    \n",
+    "    return \"\\n\".join(output) if output else \"No semantically similar transactions found.\"\n",
+    "\n",
+    "\n",
+    "SYSTEM = f\"\"\"You are a sophisticated financial analyst with access to both a SQLite database and a semantic search tool.\n",
+    "\n",
+    "Workflow:\n",
+    "1. **Identify the Need**: If the user's request uses vague terms (e.g., \"junk food\", \"travel stuff\") or you don't know the exact category/description name, start with `semantic_search`.\n",
+    "2. **Semantic Discovery**:\n",
+    "    - Call `semantic_search(query, topk=...)`. \n",
+    "    - **Pro Tip**: Use a higher `topk` (e.g., 10 or 20) if you suspect there are many relevant transactions to find.\n",
+    "    - **Relevance Check**: Not all results in the `topk` list may be relevant. You have permission to select only the few that match the user's intent and discard the rest.\n",
+    "    - **Reiteration Logic**: \n",
+    "        - If the initial results seem too narrow, or if you suspect more relevant transactions exist but were cut off, reiterate by calling `semantic_search` again with a higher `topk` (e.g., 20, 50).\n",
+    "        - **Stop Condition**: If you found a few matching transactions and are confident that's all of them (i.e., the rest of the results are clearly irrelevant), do NOT iterate further.\n",
+    "3. **Filter & Extract**:\n",
+    "    - Review the results from semantic search. Manually filter out any that aren't relevant to the user's specific intent.\n",
+    "    - Extract the unique `id` values from the metadata of relevant results.\n",
+    "4. **Handling No Results**:\n",
+    "    - If `semantic_search` returns nothing useful, or if the results are clearly not what the user asked for (e.g., user asked for \"coffee\" but results are all \"gas stations\"), STOP.\n",
+    "    - Return a clear message: \"I couldn't find any transactions related to [topic].\"\n",
+    "    - **DO NOT GUESS**. It is better to say you found nothing than to sum up unrelated transactions.\n",
+    "5. **SQL Execution**:\n",
+    "    - Use `execute_sql` to perform the final calculation or retrieval.\n",
+    "    - Correlate results by using the discovered IDs in your query: `SELECT SUM(amount) FROM transactions WHERE id IN ('uuid1', 'uuid2', ...)`.\n",
+    "    - You can also use discovered merchant names if they share a common pattern.\n",
+    "\n",
+    "Rules:\n",
+    "- **Thinking**: Explain your plan before calling any tools.\n",
+    "- **Read-only**: No modifications (INSERT/UPDATE/DELETE/etc.) to the database.\n",
+    "- **Spending Logic**: \n",
+    "    - Spending = POSITIVE values (> 0). \n",
+    "    - Payments/Refunds = NEGATIVE values (< 0). EXCLUDE negative values when calculating spending.\n",
+    "- **SQL Formatting**: Limit results to 5 rows for non-aggregation queries. Use `SUM()` for totals.\n",
+    "\"\"\"\n",
+    "\n",
+    "agent = create_agent(\n",
+    " model=vertex_llm,\n",
+    " tools=[execute_sql, semantic_search],\n",
+    " system_prompt=SYSTEM,\n",
+    " context_schema=RuntimeContext,\n",
+    " checkpointer=InMemorySaver(),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Match: CASHBACK BONUS REDEMPTION PYMT/STMT CRDT - What is this charge? Cashback bonus redemption pymt / stmt crdt . Cash - back from mogl 858-36958. His IP address, location , Internet provider, what browser he uses and operating system? You can find out all this using \"2IP spy\".It serves two main functions. It provides the network location of the host and identifies the host or network interface. What is the purpose of an IP address? Cashback bonus redemption pymt / stmt crdt .March is the month in which I had most number of transcations. I think this is because I visited the store next to me very frequently for small purchases. Cashback bonus redemption pymt / stmt crdt . Redemption Activity This Period is the total amount of Rewards you redeemed during the statement period and includes Miles partners, gift cards, account credits, electronic deposits and charitable donations. Cashback bonus redemption pymt / stmt crdt .Question2 : What was the frequency(number of occurences) of each category in terms of expenses made over the entire time period?. | Metadata: {'id': '026ac6fc-41ce-45e4-b8d5-c109b73c1e79', 'amount': -11.32, 'category': 'Awards and Rebate Credits', 'transaction_date': '2025-03-18 00:00:00', '_id': '598fb646d8a54e10a725ea61394e176a', '_collection_name': 'transactions'}\n",
+      "Match: CASHBACK BONUS REDEMPTION PYMT/STMT CRDT - What is this charge? Cashback bonus redemption pymt / stmt crdt . Cash - back from mogl 858-36958. His IP address, location , Internet provider, what browser he uses and operating system? You can find out all this using \"2IP spy\".It serves two main functions. It provides the network location of the host and identifies the host or network interface. What is the purpose of an IP address? Cashback bonus redemption pymt / stmt crdt .March is the month in which I had most number of transcations. I think this is because I visited the store next to me very frequently for small purchases. Cashback bonus redemption pymt / stmt crdt . Redemption Activity This Period is the total amount of Rewards you redeemed during the statement period and includes Miles partners, gift cards, account credits, electronic deposits and charitable donations. Cashback bonus redemption pymt / stmt crdt .Question2 : What was the frequency(number of occurences) of each category in terms of expenses made over the entire time period?. | Metadata: {'id': '8dc63630-aa67-4bb0-8012-70d244a2ba51', 'amount': -9.0, 'category': 'Awards and Rebate Credits', 'transaction_date': '2025-05-15 00:00:00', '_id': 'ecd0bafc8fee44e491fec958ffe14b51', '_collection_name': 'transactions'}\n",
+      "Match: CASHBACK BONUS REDEMPTION PYMT/STMT CRDT - What is this charge? Cashback bonus redemption pymt / stmt crdt . Cash - back from mogl 858-36958. His IP address, location , Internet provider, what browser he uses and operating system? You can find out all this using \"2IP spy\".It serves two main functions. It provides the network location of the host and identifies the host or network interface. What is the purpose of an IP address? Cashback bonus redemption pymt / stmt crdt .March is the month in which I had most number of transcations. I think this is because I visited the store next to me very frequently for small purchases. Cashback bonus redemption pymt / stmt crdt . Redemption Activity This Period is the total amount of Rewards you redeemed during the statement period and includes Miles partners, gift cards, account credits, electronic deposits and charitable donations. Cashback bonus redemption pymt / stmt crdt .Question2 : What was the frequency(number of occurences) of each category in terms of expenses made over the entire time period?. | Metadata: {'id': 'f93f111c-91bc-46f6-8a30-d14f4fb097af', 'amount': -2.55, 'category': 'Awards and Rebate Credits', 'transaction_date': '2025-11-06 00:00:00', '_id': '40f2d5f43c6d46e683de588e0ac3e36e', '_collection_name': 'transactions'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "# # Initialize the store once\n",
+    "# # vector_store = sync_to_qdrant(\"money_rag.db\")\n",
+    "\n",
+    "# Test search\n",
+    "query = \"Where did I spend money on groceries or food?\"\n",
+    "results = vector_store.similarity_search(query, k=3)\n",
+    "\n",
+    "for doc in results:\n",
+    "    print(f\"Match: {doc.page_content} | Metadata: {doc.metadata}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "================================\u001b[1m Human Message \u001b[0m=================================\n",
+      "\n",
+      "is there anything i need to be worried about in my spending over the last 2 months?\n",
+      "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
+      "Tool Calls:\n",
+      "  execute_sql (53fdebc4-4abc-4bda-ac02-e1f2f89b8140)\n",
+      " Call ID: 53fdebc4-4abc-4bda-ac02-e1f2f89b8140\n",
+      "  Args:\n",
+      "    query: SELECT SUM(amount) FROM transactions WHERE transaction_date >= date('now', '-2 months') AND amount > 0\n",
+      "=================================\u001b[1m Tool Message \u001b[0m=================================\n",
+      "Name: execute_sql\n",
+      "\n",
+      "[(2028.51,)]\n",
+      "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
+      "\n",
+      "Your total spending over the last two months is $2028.51.\n",
+      "\n",
+      "To tell you if there's anything to \"worry about,\" I need a bit more information. Would you like me to:\n",
+      "1.  Break down your spending by category?\n",
+      "2.  Identify any unusually large transactions?\n",
+      "3.  Compare your spending to previous months?\n",
+      "4.  Look for any recurring subscriptions?\n"
+     ]
+    }
+   ],
+   "source": [
+    "question = \"is there anything i need to be worried about in my spending over the last 2 months?\"\n",
+    "steps = []\n",
+    "\n",
+    "for step in agent.stream(\n",
+    "    {\"messages\": [{\"role\": \"user\", \"content\": question}]},\n",
+    "    {\"configurable\": {\"thread_id\": \"1\"}},\n",
+    "    stream_mode=\"values\",\n",
+    "    context=RuntimeContext(db=db, vector_store=vector_store)\n",
+    "):\n",
+    "    step[\"messages\"][-1].pretty_print()\n",
+    "    steps.append(step)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💬 Chat with your financial data! (Type 'exit' to stop)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# ...existing code...\n",
+    "# Interactive Chat Loop\n",
+    "print(\"💬 Chat with your financial data! (Type 'exit' to stop)\")\n",
+    "\n",
+    "while True:\n",
+    "    try:\n",
+    "        user_input = input(\"User: \")\n",
+    "        if user_input.lower() in [\"exit\", \"quit\", \"q\"]:\n",
+    "            print(\"Goodbye!\")\n",
+    "            break\n",
+    "            \n",
+    "        print(\"\\n\" + \"-\"*50)\n",
+    "        \n",
+    "        # Stream the agent's response\n",
+    "        for step in agent.stream(\n",
+    "            {\"messages\": [{\"role\": \"user\", \"content\": user_input}]},\n",
+    "            {\"configurable\": {\"thread_id\": \"1\"}}, # Keeps memory of the conversation\n",
+    "            stream_mode=\"values\",\n",
+    "            context=RuntimeContext(db=db, vector_store=vector_store)\n",
+    "        ):\n",
+    "            step[\"messages\"][-1].pretty_print()\n",
+    "            \n",
+    "        print(\"-\" * 50 + \"\\n\")\n",
+    "        \n",
+    "    except KeyboardInterrupt:\n",
+    "        print(\"\\nGoodbye!\")\n",
+    "        break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
   {
    "cell_type": "code",
    "execution_count": null,

requirements.txt

@@ -6,4 +6,6 @@ tabulate
 langchain-google-vertexai
 mcp 
 langchain-mcp
-langchain-community
+langchain-community
+qdrant-client
+langchain-qdrant