{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search Sermon Chunks\n",
    "This notebook allows you to load and search through the parsed sermon chunks. The chunk metadata (like `date_code`) has been corrected and is fully reliable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import re\n",
    "import pandas as pd\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# Load the corrected sermon chunks\n",
    "with open('sermon_chunks.pkl', 'rb') as f:\n",
    "    chunks = pickle.load(f)\n",
    "\n",
    "print(f\"Loaded {len(chunks)} sermon chunks.\")\n",
    "if chunks:\n",
    "    print(f\"Example chunk type: {type(chunks[0])}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. General Text Search\n",
    "Search for a keyword or phrase anywhere in the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_text(query, chunks, top_k=10, search_metadata=False):\n",
    "    \"\"\"\n",
    "    Search through the chunks for a given text query.\n",
    "    \"\"\"\n",
    "    results = []\n",
    "    query_lower = query.lower()\n",
    "    \n",
    "    for chunk in chunks:\n",
    "        content = chunk.page_content\n",
    "        match_found = query_lower in content.lower()\n",
    "        \n",
    "        if not match_found and search_metadata and hasattr(chunk, 'metadata'):\n",
    "            meta = chunk.metadata\n",
    "            title = str(meta.get('title', '')).lower()\n",
    "            source = str(meta.get('source', '')).lower()\n",
    "            if query_lower in title or query_lower in source:\n",
    "                match_found = True\n",
    "                \n",
    "        if match_found:\n",
    "            content_snippet = content[:300] + '...'\n",
    "            results.append({\n",
    "                'Title': chunk.metadata.get('title'),\n",
    "                'Date': chunk.metadata.get('date_code'),\n",
    "                'Paragraph': chunk.metadata.get('paragraph'),\n",
    "                'Page Range': f\"{chunk.metadata.get('page_start')} - {chunk.metadata.get('page_end')}\",\n",
    "                'Snippet': content_snippet\n",
    "            })\n",
    "            \n",
    "            if len(results) >= top_k:\n",
    "                break\n",
    "            \n",
    "    return pd.DataFrame(results)\n",
    "\n",
    "# Example usage:\n",
    "# display(search_text(\"faith\", chunks, top_k=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Search by Exact Paragraph\n",
    "Find a specific paragraph using the corrected `date_code` metadata."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_paragraph(date_code, paragraph_num, chunks):\n",
    "    \"\"\"\n",
    "    Find a specific paragraph in a specific sermon.\n",
    "    Now relies purely on the corrected `date_code` metadata.\n",
    "    \n",
    "    Args:\n",
    "        date_code (str): The date code of the sermon, e.g., '53-0405S' or '55-0123A'\n",
    "        paragraph_num (str/int): The paragraph number, e.g., '15' or 15\n",
    "        chunks: The loaded list of Document chunks\n",
    "    \"\"\"\n",
    "    results = []\n",
    "    target_date = str(date_code).strip().upper()\n",
    "    target_para = str(paragraph_num).strip()\n",
    "    \n",
    "    for chunk in chunks:\n",
    "        meta = getattr(chunk, 'metadata', {})\n",
    "        \n",
    "        chunk_date = str(meta.get('date_code', '')).strip().upper()\n",
    "        chunk_para = str(meta.get('paragraph', '')).strip()\n",
    "        \n",
    "        # Clean direct match\n",
    "        if chunk_date == target_date and chunk_para == target_para:\n",
    "            results.append({\n",
    "                'Chunk ID': meta.get('chunk_id'),\n",
    "                'Date Code': chunk_date,\n",
    "                'Source': meta.get('source'),\n",
    "                'Paragraph': chunk_para,\n",
    "                'Page Range': f\"{meta.get('page_start')} - {meta.get('page_end')}\",\n",
    "                'Title': meta.get('title'),\n",
    "                'Content': chunk.page_content\n",
    "            })\n",
    "            \n",
    "    if not results:\n",
    "        print(f\"No results found for Date: {target_date}, Paragraph: {target_para}\")\n",
    "        \n",
    "    return pd.DataFrame(results)\n",
    "\n",
    "# --- Example Query ---\n",
    "target_date = \"55-0123A\"\n",
    "target_paragraph = \"74\"\n",
    "\n",
    "para_df = find_paragraph(target_date, target_paragraph, chunks)\n",
    "\n",
    "if not para_df.empty:\n",
    "    print(\"--- FULL PARAGRAPH CONTENT ---\")\n",
    "    print(para_df.iloc[0]['Content'])\n",
    "    print(\"\\n--- METADATA ---\")\n",
    "    display(para_df.drop(columns=['Content']))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Advanced Regex Search\n",
    "Use regular expressions to find complex patterns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regex_search(pattern, chunks, top_k=10, flags=re.IGNORECASE):\n",
    "    \"\"\"\n",
    "    Search through chunks using a Regular Expression pattern.\n",
    "    \"\"\"\n",
    "    results = []\n",
    "    regex = re.compile(pattern, flags)\n",
    "    \n",
    "    for chunk in chunks:\n",
    "        content = chunk.page_content\n",
    "        if regex.search(content):\n",
    "            # Highlight match in snippet (simple truncation for display)\n",
    "            match_idx = content.lower().find(pattern.lower().replace('\\\\b', '')) if not '\\\\' in pattern else 0\n",
    "            start_idx = max(0, match_idx - 50)\n",
    "            snippet = \"...\" + content[start_idx:start_idx + 300].replace('\\n', ' ') + \"...\"\n",
    "            \n",
    "            results.append({\n",
    "                'Date': chunk.metadata.get('date_code'),\n",
    "                'Paragraph': chunk.metadata.get('paragraph'),\n",
    "                'Snippet': snippet\n",
    "            })\n",
    "            if len(results) >= top_k:\n",
    "                break\n",
    "                \n",
    "    return pd.DataFrame(results)\n",
    "\n",
    "# Example: find word 'eagle' followed eventually by 'wings'\n",
    "# display(regex_search(r'eagle.*wings', chunks, top_k=5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}