Spaces:
Running on Zero
Running on Zero
File size: 3,774 Bytes
d10de1b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | {
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# NLP4ASD – Build Knowledge Base\n",
"\n",
"Run this notebook to:\n",
"1. Load raw documents from `data/raw/`\n",
"2. Clean and chunk them\n",
"3. Embed with SentenceTransformers\n",
"4. Build and save the FAISS index\n",
"\n",
"**Run this after adding or changing documents.**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os, sys\n",
"# Add project root to path\n",
"sys.path.insert(0, os.path.abspath('..'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from src.data_loader import load_all_documents\n",
"from src.config import DATA_RAW_DIR\n",
"\n",
"docs = load_all_documents(DATA_RAW_DIR)\n",
"print(f\"Loaded {len(docs)} documents\")\n",
"for d in docs:\n",
" print(f\" - {d['source']} ({len(d['text'])} chars)\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from src.preprocessing import preprocess_documents\n",
"\n",
"docs_clean = preprocess_documents(docs)\n",
"print(\"Sample (first 300 chars of doc 0):\")\n",
"print(docs_clean[0]['text'][:300])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from src.chunking import chunk_documents\n",
"\n",
"chunks = chunk_documents(docs_clean)\n",
"print(f\"Total chunks: {len(chunks)}\")\n",
"print(\"\\nSample chunk:\")\n",
"print(chunks[5])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from src.embeddings import embed_texts\n",
"\n",
"texts = [c['text'] for c in chunks]\n",
"embeddings = embed_texts(texts)\n",
"print(f\"Embeddings shape: {embeddings.shape}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from src.vector_store import build_index, save_index\n",
"\n",
"index = build_index(embeddings)\n",
"save_index(index, chunks)\n",
"print(\"Index saved successfully!\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Test retrieval\n",
"from src.retriever import retrieve, reset_index_cache\n",
"\n",
"reset_index_cache() # ensure fresh load after rebuild\n",
"\n",
"query = \"What are early signs of autism?\"\n",
"results = retrieve(query, top_k=3)\n",
"\n",
"print(f\"Query: {query}\\n\")\n",
"for i, r in enumerate(results, 1):\n",
" print(f\"[Result {i}] Source: {r['source']} | Score: {r['score']:.4f}\")\n",
" print(r['text'][:200])\n",
" print()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Full pipeline end-to-end test\n",
"from src.rag_pipeline import answer\n",
"\n",
"result = answer(\n",
" question=\"What are the most effective interventions for autistic children?\",\n",
" profile=\"Parent\",\n",
" language=\"English\"\n",
")\n",
"\n",
"print(\"ANSWER:\\n\", result['answer'])\n",
"print(\"\\nSOURCES:\\n\", result['sources'])\n",
"print(\"\\nDISCLAIMER:\\n\", result['disclaimer'])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.10.0"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|