{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP4ASD – Build Knowledge Base\n",
    "\n",
    "Run this notebook to:\n",
    "1. Load raw documents from `data/raw/`\n",
    "2. Clean and chunk them\n",
    "3. Embed with SentenceTransformers\n",
    "4. Build and save the FAISS index\n",
    "\n",
    "**Run this after adding or changing documents.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "# Add project root to path\n",
    "sys.path.insert(0, os.path.abspath('..'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data_loader import load_all_documents\n",
    "from src.config import DATA_RAW_DIR\n",
    "\n",
    "docs = load_all_documents(DATA_RAW_DIR)\n",
    "print(f\"Loaded {len(docs)} documents\")\n",
    "for d in docs:\n",
    "    print(f\"  - {d['source']} ({len(d['text'])} chars)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.preprocessing import preprocess_documents\n",
    "\n",
    "docs_clean = preprocess_documents(docs)\n",
    "print(\"Sample (first 300 chars of doc 0):\")\n",
    "print(docs_clean[0]['text'][:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.chunking import chunk_documents\n",
    "\n",
    "chunks = chunk_documents(docs_clean)\n",
    "print(f\"Total chunks: {len(chunks)}\")\n",
    "print(\"\\nSample chunk:\")\n",
    "print(chunks[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.embeddings import embed_texts\n",
    "\n",
    "texts = [c['text'] for c in chunks]\n",
    "embeddings = embed_texts(texts)\n",
    "print(f\"Embeddings shape: {embeddings.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.vector_store import build_index, save_index\n",
    "\n",
    "index = build_index(embeddings)\n",
    "save_index(index, chunks)\n",
    "print(\"Index saved successfully!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test retrieval\n",
    "from src.retriever import retrieve, reset_index_cache\n",
    "\n",
    "reset_index_cache()  # ensure fresh load after rebuild\n",
    "\n",
    "query = \"What are early signs of autism?\"\n",
    "results = retrieve(query, top_k=3)\n",
    "\n",
    "print(f\"Query: {query}\\n\")\n",
    "for i, r in enumerate(results, 1):\n",
    "    print(f\"[Result {i}] Source: {r['source']} | Score: {r['score']:.4f}\")\n",
    "    print(r['text'][:200])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Full pipeline end-to-end test\n",
    "from src.rag_pipeline import answer\n",
    "\n",
    "result = answer(\n",
    "    question=\"What are the most effective interventions for autistic children?\",\n",
    "    profile=\"Parent\",\n",
    "    language=\"English\"\n",
    ")\n",
    "\n",
    "print(\"ANSWER:\\n\", result['answer'])\n",
    "print(\"\\nSOURCES:\\n\", result['sources'])\n",
    "print(\"\\nDISCLAIMER:\\n\", result['disclaimer'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}