prototype stage

.gitignore

@@ -0,0 +1,108 @@
+# -------------------------------------------------------------------------
+# 1. Python & Virtual Environments (Standard)
+# -------------------------------------------------------------------------
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.venv/
+venv/
+env/
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# -------------------------------------------------------------------------
+# 2. Security & Secrets (CRITICAL)
+# Never commit your API keys (HuggingFace, OpenAI, etc.)
+# -------------------------------------------------------------------------
+.env
+.env.local
+.env.*
+secrets.json
+credentials.json
+
+# -------------------------------------------------------------------------
+# 3. Large AI Models & Weights (Hugging Face / PyTorch)
+# These files are GBs in size; use DVC to track them instead.
+# -------------------------------------------------------------------------
+*.bin
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+*.onnx
+models/
+weights/
+checkpoints/
+lora-adapters/
+
+# -------------------------------------------------------------------------
+# 4. Data & RAG Stores (MshauriFedha Specific)
+# Ignore raw PDFs and local vector databases (Chroma/Faiss).
+# -------------------------------------------------------------------------
+data/
+datasets/
+corpus/
+# Ignore ChromaDB and Faiss local persistence folders
+chroma_db/
+chroma_storage/
+faiss_indexes/
+storage/ 
+
+# -------------------------------------------------------------------------
+# 5. CSCS & HPC Specifics
+# Ignore the huge container images and Slurm log files.
+# -------------------------------------------------------------------------
+*.sif
+*.sif.*
+*.tar.gz
+slurm-*.out
+slurm-*.err
+core.*
+
+# -------------------------------------------------------------------------
+# 6. Jupyter Notebooks
+# -------------------------------------------------------------------------
+.ipynb_checkpoints/
+*-checkpoint.ipynb
+# Optional: if you don't want to commit notebook outputs (just code)
+# *.ipynb (Uncomment this if you only want to commit .py scripts)
+
+# -------------------------------------------------------------------------
+# 7. DVC (Data Version Control)
+# We ignore the local cache/config but KEEP the .dvc files.
+# -------------------------------------------------------------------------
+/dvc_storage
+.dvc/config.local
+.dvc/tmp
+.dvc/cache
+
+# -------------------------------------------------------------------------
+# 8. IDE & Editors
+# -------------------------------------------------------------------------
+.vscode/
+.idea/
+.DS_Store
+
+# -------------------------------------------------------------------------
+# 9. Docker / Deployment
+# -------------------------------------------------------------------------
+docker-compose.override.yml
+.coverage
+htmlcov/
+*.err
+*.out

notebooks/inspect_md.ipynb

@@ -0,0 +1,426 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "62db404a-4930-4279-afa2-35ae4d11d857",
+   "metadata": {},
+   "source": [
+    "# Inspect markdown files - KNBS and CBK\n",
+    "In this notebook, the core objective is to inspect and ingest the text from the already processed markdown files for CBK and KNBS."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e8696cf9-6995-4af3-937f-9154ee6d0a99",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "44ff42b3-2377-4e1e-accb-02706eaae797",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import logging\n",
+    "import warnings\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "warnings.filterwarnings(\"ignore\")\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Configure logging to see output in the notebook\n",
+    "logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0d114027-fee3-4186-9d02-f6535c728553",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fix paths for src files\n",
+    "project_root = Path(os.getcwd()).parent\n",
+    "script_dir = project_root / \"src\"\n",
+    "if str(script_dir) not in sys.path:\n",
+    "    sys.path.append(str(script_dir))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "64a8246f-58d2-4aa7-bd2c-c6a2741f8c19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.start_ollama import start_ollama_server\n",
+    "from load.ingest_md import ingest_markdown_reports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e6fa2ada-beac-448a-b1d2-535fd2b5d0b1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Ollama is already running.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "start_ollama_server()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "894e6598-3c57-4484-bc3c-be043f06b5ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Define your paths\n",
+    "SCRATCH_DIR = os.environ.get(\"SCRATCH\")\n",
+    "KNBS_MARKDOWN_DIR = os.path.join(SCRATCH_DIR, \"mshauri-fedha/data/knbs/marker-output\")\n",
+    "VECTOR_DB_PATH = \"mshauri_fedha_chroma_db\"\n",
+    "EMBEDDING_MODEL = \"nomic-embed-text\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "d8b23cd1-ea3f-4f7b-b553-dbeca35ee61a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "📄 Scanning for Markdown Reports in /capstor/scratch/cscs/tligawa/mshauri-fedha/data/knbs/marker-output...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 574/574 [00:00<00:00, 4626.46it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   Loaded 574 report files.\n",
+      "   ✂️  Split into 32717 chunks.\n",
+      "🧠 Appending to Vector Store...\n",
+      "INFO:chromadb.telemetry.product.posthog:Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Ingesting Reports: 100%|██████████| 32717/32717 [1:09:38<00:00,  7.83chunk/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✅ Reports Added. Hybrid Knowledge Base is ready.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run the ingestion on KNBS text\n",
+    "ingest_markdown_reports(\n",
+    "    markdown_dir=KNBS_MARKDOWN_DIR,\n",
+    "    vector_db_path=VECTOR_DB_PATH,\n",
+    "    model=EMBEDDING_MODEL\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "95eae0fe-f71f-475d-9944-2f202a30174c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_community.embeddings import OllamaEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "72488cf7-3778-481a-8673-b7b08dd28e5a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "CBK_MARKDOWN_DIR = os.path.join(SCRATCH_DIR, \"mshauri-fedha/data/cbk/marker-output\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "8b6ef094-487a-4148-9435-1e9546d6f5a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "📄 Scanning for Markdown Reports in /capstor/scratch/cscs/tligawa/mshauri-fedha/data/cbk/marker-output...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 958/958 [00:11<00:00, 79.89it/s] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   Loaded 958 report files.\n",
+      "   ✂️  Split into 4582 chunks.\n",
+      "🧠 Appending to Vector Store...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Ingesting Reports: 100%|██████████| 4582/4582 [10:21<00:00,  7.37chunk/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✅ Reports Added. Hybrid Knowledge Base is ready.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run the ingestion on CBK text\n",
+    "ingest_markdown_reports(\n",
+    "    markdown_dir=CBK_MARKDOWN_DIR,\n",
+    "    vector_db_path=VECTOR_DB_PATH,\n",
+    "    model=EMBEDDING_MODEL\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d85e55c2-1f09-4550-b2d2-4ae5ccae5ae3",
+   "metadata": {},
+   "source": [
+    "## Test the performance\n",
+    "Using similarity search, we test the performance of the embedding model used here."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "6aece829-bceb-498b-a549-cf29287b1bea",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔎 Checking for tables in: 'Interest rates commercial banks 2020'\n",
+      "\n",
+      "---------------------------------------------------\n",
+      "📰 Source: knbs_batch_42_193220_2020kenyafactsfigures\n",
+      "| Value of shares traded (KSh Bn)         | 209   | 147   | 172   | 176   | 154   |\n",
+      "| Equities market capitalization (KSh Bn) | 2,054 | 1,932 | 2,522 | 2,102 | 2,540 |\n",
+      "| NSE 20 Share Index (Base Jan 1966=100)  | 4,040 | 3,186 | 3,712 | 2,834 | 2,654 |\n",
+      "\n",
+      "<sup>1</sup> W eig hted av erag e commercial bank interes t rates\n",
+      "\n",
+      "# **Monetary and Financial Statistics**\n",
+      "\n",
+      "**Table 21: Commercial Banks' Deposits, Loans and Advances, 2015 - 2019**\n",
+      "\n",
+      "| 2015                                           | 2016        | 2017        | 2018        | 2019*       |             |\n",
+      "|------------------------------------------------|-------------|-------------|-------------|-------------|-------------|\n",
+      "| Commercial bank<br>s (KS<br>h million)         |             |             |             |             |             |\n",
+      "| Deposits<br>liabilities                        | 2,661,140.0 | 2,771,710.6 | 3,068,723.8 | 3,414,705.5 | 3,634,995.9 |\n",
+      "| Total loans<br>and advances                    | 2,873,799.6 | 3,127,888.0 | 3,318,907.0 | 3,543,932.0 | 3,838,796.6 |\n",
+      "| Public sector                                  | 630,049.0   | 814,585.0   | 930,174.5   | 1,057,217.2 | 1,177,091.3 |\n",
+      "| Private sector                                 | 2,243,750.7 | 2,317,025.0 | 2,415,922.9 | 2,486,714.9 | 2,664,382.5 |\n",
+      "| Number of authorised institutions in operation |             |             |             |             |             |\n",
+      "| Licensed banks                                 | 43          | 43          | 42          | 42 ...\n",
+      "\n",
+      "---------------------------------------------------\n",
+      "📰 Source: knbs_batch_42_193220_thekenyapovertyreport2020\n",
+      "| Interest rate on commercial bank loans and advances $(\\%)$                                                                                                                                                                                                                                               | 17.44                                                                                                       | 2.607                                                                                                | 1                                                                                                    | 9                                                                                                |                                                                                                       | 12.02                                                                                                |\n",
+      "| Formal Employment sector (000's)                                                                                                                                                                                                                                                                         | 2,601                                                                                                       |                                                                                                      | ,,,,                                                           ...\n",
+      "\n",
+      "---------------------------------------------------\n",
+      "📰 Source: knbs_batch_35_193128_2021kenyafactsfigures\n",
+      "| 100 Japanese Yen                | 93.55  | 92.22  | 91.74  | 93.59  | 99.80  |\n",
+      "| 1 SA Rand                       | 6.93   |        |        |        |        |\n",
+      "| KSh /TSh                        | 21.54  | 21.63  | 22.48  | 22.63  | 21.76  |\n",
+      "| KSh/ Ush                        | 33.68  | 34.92  | 36.81  | 36.32  | 34.93  |\n",
+      "| Overall Weighted Index 2009=100 | 114.30 | 114.83 | 116.52 | 115.66 | 113.04 |\n",
+      "\n",
+      "<sup>\\*</sup>Provisional\n",
+      "\n",
+      "<sup>2</sup> Countries in the Euro area included in the computation of Trade Weighted Fisher's Ideal Index are: Germany, France, Switzerland, Netherlands, Belgium and Italy.\n",
+      "\n",
+      "**Table 19: Nominal Interest Rates, 2016 – 2020**\n",
+      "\n",
+      "Percentage **2016 2017 2018 2019 2020** 91-day Treasury bill rate................................... 8.44 8.01 7.34 7.17 6.90 Inter-Bank Offered Rate.................................... 5.92 7.27 8.15 6.03 5.29 Overdraft Rates.................................................. 13.49 13.54 12.17 11.67 11.51 Commercial Banks Loans and Advances...... 13.69 13.64 12.51 12.24 12.02 Savings deposits rate....................................... 6.37 6.91 5.13 4.02 2.70\n",
+      "\n",
+      "<sup>1</sup>Weighted average commercial bank interest rates\n",
+      "\n",
+      "Table 20: Securities Exchange, 2016 - 2020\n",
+      "\n",
+      "| 2016                                    | 2017  | 2018  | 2019  | 2020* |       |\n",
+      "|-----------------------------------------|-------|-------|-------|-------|-------|\n",
+      "| Value of shares traded (KSh Bn)         | 147   | 172   | 176   | 154   | 149   |\n",
+      "| Equities Market c...\n",
+      "\n",
+      "---------------------------------------------------\n",
+      "📰 Source: knbs_batch_32_193040_2025_facts_and_figures\n",
+      "| Kenya Development Corporation2                    | -    | 10                 | 9    | 4    | 7     | -       | 521.4   | 613.8                         | 510.0   | 599.9   |\n",
+      "| Industrial and Commercial Development Corporation | 3    | -                  | -    | -    | -     | 100.9   | -       | -                             | -       | -       |\n",
+      "| Sub - total                                       | 320  | 328                | 303  | 362  | 343   | 1,096.3 | 1,394.5 | 1,690.8                       | 1,869.9 | 2,304.9 |\n",
+      "| All other commercial banks1                       | -    | -                  | -    | -    | -     | 410,640 | 463,981 | 527,235                       | 637,513 | 560,643 |\n",
+      "| TOTAL                                             | 320  | 328                | 303  | 362  | 343   | 411,736 | 465,376 | 528,926                       | 639,383 | 562,948 |\n",
+      "\n",
+      "*<sup>\\*</sup> Provisional* \n",
+      "\n",
+      "*<sup>1</sup> Source: Central Bank of Kenya (excludes DBK).* \n",
+      "\n",
+      "*<sup>2</sup> IDB Capital, Tourism Finance Corporation and ICDC merged to form KDC in 2020* \n",
+      "\n",
+      "![](_page_92_Picture_0.jpeg)\n",
+      "\n",
+      "**Table 43: Selected EPZ Performance Indicators, 2020 - 2024**\n",
+      "\n",
+      "|                                          | Unit           | 2020    | 2021    | 2022    | 2023    | 2024*   |\n",
+      "|------------------------------------------|----------------|---------|---------|---------|---------|---------|\n",
+      "| Gazetted Zones                           | Number         | 76      | 82      | 89      | 102     | 105     |\n",
+      "| ...\n",
+      "\n",
+      "---------------------------------------------------\n",
+      "📰 Source: knbs_batch_34_193116_2022economicsurvey\n",
+      "| Savings deposits.           | -       | -        | -     | -     | -     | -     |      |\n",
+      "| Loan and Advances (maximum) | 13.25   | 12.10    | 12.19 | 12.04 | 12.04 | 12.16 |      |\n",
+      "| Overdraft.                  | -       | -        | -     | -     | -     | -     |      |\n",
+      "| Loans-Deposits Spread       | -       | -        | -     | -     | 5.69  | -     |      |\n",
+      "\n",
+      "*Source: Central Bank of Kenya.*\n",
+      "\n",
+      "*Selected financial aggregates values are deflated using December Consumer Price Indices*\n",
+      "\n",
+      "*Weighted average commercial bank interest rates*\n",
+      "\n",
+      "*<sup>\\*</sup>Provisional*\n",
+      "\n",
+      "**4.12.** Table 4.7 shows the selected real principal interest rates from 2017 to 2021. Real interest rates reflect the real cost of borrowing, savings and return on investment. The weighted average real interest rate for commercial bank deposits increased to 0.77 per cent in 2021 from 0.68 per cent in 2020. Commercial banks loans and advances rate increased from 6.40 per cent in December 2020 to 6.43 per cent in December 2021. The real average interest rate for the 91-day Treasury Bills increased from 1.28 per cent in December 2020 to 1.53 per cent in December 2021 while the inter-bank rate declined further from negative 0.33 per cent in December 2020 to negative 0.63 per cent in December 2021.\n",
+      "\n",
+      "**Table 4.7: Selected Real Principal Interest Rates, 2017 – 2021**\n",
+      "\n",
+      "*Per cent*\n",
+      "\n",
+      "| Average Interest Rate for 91-day Treasury Bills | Year  | Nominal Interest | Inflation Rate | Real Interest1 |\n",
+      "|---------------------------...\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Connect\n",
+    "embeddings = OllamaEmbeddings(model=EMBEDDING_MODEL, base_url=\"http://127.0.0.1:25000\")\n",
+    "vectorstore = Chroma(persist_directory=VECTOR_DB_PATH, embedding_function=embeddings)\n",
+    "\n",
+    "# Query for a known table\n",
+    "query = \"Interest rates commercial banks 2020\" \n",
+    "\n",
+    "results = vectorstore.similarity_search(query, k=5)\n",
+    "\n",
+    "print(f\"🔎 Checking for tables in: '{query}'\\n\")\n",
+    "\n",
+    "for i, doc in enumerate(results):\n",
+    "    content = doc.page_content\n",
+    "    print(\"---------------------------------------------------\")\n",
+    "    print(f\"Source: {doc.metadata.get('source', 'N/A').split('/')[-1].split('.')[0].replace('-', '')}\")\n",
+    "    # Print first 500 chars to see if headers align\n",
+    "    print(content[:1500] + \"...\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc9072f0-fcd2-4b24-9050-795e64b26ff6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/inspect_news.ipynb

@@ -0,0 +1,476 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4b0cd97a-60f0-4582-a8c3-b4d9dbf7ab03",
+   "metadata": {},
+   "source": [
+    "# Inspect Business News\n",
+    "In this notebook, the study will seek to inspect news files (stored as csv files), and ingest its content to the vectordb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ed41b128-7baa-43e2-9b1a-2cfdc955440a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "c80c78f8-8118-46b5-b954-e0b15d935ced",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import glob\n",
+    "import os\n",
+    "import sys\n",
+    "import warnings\n",
+    "from pathlib import Path\n",
+    "warnings.filterwarnings(\"ignore\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "7801b40d-c6a9-44cd-8afb-e544731aeb52",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fix paths for src files\n",
+    "project_root = Path(os.getcwd()).parent\n",
+    "script_dir = project_root / \"src\"\n",
+    "if str(script_dir) not in sys.path:\n",
+    "    sys.path.append(str(script_dir))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2f940902-b38d-4e6e-9566-55f511dc0bc9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.explore_news_schema import analyze_schemas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "788c9e1d-5c17-4387-9153-77a31cd26eec",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔍 Scanning 22 files in '/capstor/scratch/cscs/tligawa/mshauri-fedha/data/news'...\n",
+      "\n",
+      "--- Schema Report ---\n",
+      "\n",
+      "TYPE 1: Found in 5 files\n",
+      "Columns: ['description', 'published date', 'publisher', 'title', 'url']\n",
+      "Examples: ['google_news_10-11-2025.csv', 'google_news_10-11-2025-19-27.csv', 'google_news_19-11-2025-19-49.csv'] ... (+2 others)\n",
+      "\n",
+      "TYPE 2: Found in 7 files\n",
+      "Columns: ['authors', 'date', 'full_content', 'image', 'source', 'summary', 'title', 'url', 'word_count']\n",
+      "Examples: ['kenya_news_full_27-10-2025.csv', 'kenya_news_full_17-11-2025-17-52.csv', 'newsdata_10-11-2025.csv'] ... (+4 others)\n",
+      "\n",
+      "TYPE 3: Found in 10 files\n",
+      "Columns: ['content', 'date', 'source', 'title', 'url']\n",
+      "Examples: ['gnews_19-11-2025-19-49.csv', 'the_news_10-11-2025.csv', 'the_news_19-11-2025-19-49.csv'] ... (+7 others)\n",
+      "\n",
+      "--- Date Format Sample ---\n",
+      "Sample from column 'published date' in google_news_10-11-2025.csv:\n",
+      "['Sun, 09 Nov 2025 03:15:00 GMT', 'Sun, 09 Nov 2025 18:45:00 GMT', 'Tue, 04 Nov 2025 06:00:00 GMT', 'Tue, 04 Nov 2025 17:33:08 GMT', 'Wed, 05 Nov 2025 09:38:38 GMT']\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Path to where you just downloaded the files\n",
+    "SCRATCH_DIR = os.environ.get(\"SCRATCH\")\n",
+    "NEWS_DIR = os.path.join(SCRATCH_DIR, \"mshauri-fedha/data/news\")\n",
+    "\n",
+    "# Run the exploration\n",
+    "analyze_schemas(NEWS_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "f65ee0d4-2bac-46f4-8cd4-44ac9c903ee3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.ingest_news import ingest_news_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "c7864bbb-e731-45b4-bd9e-4dbf9a063653",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.start_ollama import start_ollama_server, pull_embedding_model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "279c443b-4502-40d9-9b29-06e1929ba842",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🚀 Starting Ollama Server...\n",
+      "⏳ Waiting for server to boot...\n",
+      "✅ Server started successfully.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "start_ollama_server()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "03b5c2c0-4d6a-4e06-ab88-806ca5eaa2d6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⬇️  Requesting pull for 'nomic-embed-text'...\n",
+      "   success manifest digest00%\n",
+      "✅ Model 'nomic-embed-text' installed successfully!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# pull embedding model\n",
+    "pull_embedding_model(\"nomic-embed-text\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "3a4a7e63-934c-4b0d-824f-f525278139dc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🚀 Found 22 news files. Processing...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Reading CSVs: 100%|██████████| 22/22 [00:00<00:00, 179.19file/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   📉 Condensed into 198 unique articles.\n",
+      "🧠 Embedding 455 chunks into Vector DB...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "/users/tligawa/mshauri-fedha/notebooks/ingest_news.py:149: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the `langchain-ollama package and should be used instead. To use it run `pip install -U `langchain-ollama` and import as `from `langchain_ollama import OllamaEmbeddings``.\n",
+      "  embeddings = OllamaEmbeddings(model=model, base_url=\"http://127.0.0.1:25000\")\n",
+      "/users/tligawa/mshauri-fedha/notebooks/ingest_news.py:150: LangChainDeprecationWarning: The class `Chroma` was deprecated in LangChain 0.2.9 and will be removed in 1.0. An updated version of the class exists in the `langchain-chroma package and should be used instead. To use it run `pip install -U `langchain-chroma` and import as `from `langchain_chroma import Chroma``.\n",
+      "  vectorstore = Chroma(persist_directory=vector_db_path, embedding_function=embeddings)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Embedding News: 100%|██████████| 455/455 [00:21<00:00, 21.06chunk/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✅ News Ingestion Complete.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "VECTOR_DB = \"mshauri_fedha_chroma_db\"\n",
+    "EMBEDDING_MODEL = \"nomic-embed-text\" # Make sure this matches your existing DB model\n",
+    "\n",
+    "# Run\n",
+    "ingest_news_data(NEWS_DIR, VECTOR_DB, EMBEDDING_MODEL)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d993f352-44a4-4c73-ae2f-cc3b2e85a882",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_community.embeddings import OllamaEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee969dc7-6ecb-45e3-ab89-875536177543",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# --- CONFIG ---\n",
+    "VECTOR_DB_PATH = \"mshauri_fedha_chroma_db\"\n",
+    "EMBEDDING_MODEL = \"nomic-embed-text\"\n",
+    "OLLAMA_URL = \"http://127.0.0.1:25000\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e681dd43-f0f3-47ec-8445-4d188eb7886a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Connect to DB\n",
+    "print(\"Connecting to Vector Store...\")\n",
+    "embeddings = OllamaEmbeddings(model=EMBEDDING_MODEL, base_url=OLLAMA_URL)\n",
+    "vectorstore = Chroma(persist_directory=VECTOR_DB_PATH, embedding_function=embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5decfff3-26d7-4784-b6b0-7fd98a34f3ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Get Stats\n",
+    "count = vectorstore._collection.count()\n",
+    "print(f\"Total Documents stored: {count}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "d55aab85-10c7-44a0-83cf-69c125416b61",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔌 Connecting to Vector Store...\n",
+      "✅ Total Documents stored: 455\n",
+      "\n",
+      "👀 Random Sample Document:\n",
+      "--- Metadata ---\n",
+      "Date:   2025-11-03\n",
+      "Source: african markets\n",
+      "Type:   news\n",
+      "\n",
+      "--- Content (First 300 chars) ---\n",
+      "Title: BGFI Holding finally gets regulatory approval for its BVMAC IPO: inside a tumultuous IPO journey - african markets\n",
+      "Date: 2025-11-03\n",
+      "Source: african markets\n",
+      "\n",
+      "BGFI Holding finally gets regulatory approval for its BVMAC IPO: inside a tumultuous IPO journey african markets...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Peek at a Sample\n",
+    "print(\"\\n Random Sample Document:\")\n",
+    "# We fetch 1 random ID just to peek\n",
+    "result = vectorstore.get(limit=1)\n",
+    "\n",
+    "if result['ids']:\n",
+    "    meta = result['metadatas'][0]\n",
+    "    content = result['documents'][0]\n",
+    "    \n",
+    "    print(f\"--- Metadata ---\")\n",
+    "    print(f\"Date:   {meta.get('date')}\")\n",
+    "    print(f\"Source: {meta.get('source')}\")\n",
+    "    print(f\"Type:   {meta.get('type')}\")\n",
+    "    \n",
+    "    print(f\"\\n--- Content (First 300 chars) ---\")\n",
+    "    print(content[:300] + \"...\")\n",
+    "else:\n",
+    "    print(\"Database is empty!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "f1192ee8-f09a-4e11-84df-aacea487485e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "🔎 Searching for: 'How have the protests impacted the Kenyan economy?'...\n",
+      "Found 10 relevant articles:\n",
+      "\n",
+      "Result #1 -------------------------\n",
+      "📅 Date:   2025-09-29\n",
+      "📰 Source: Devdiscourse\n",
+      "📝 Excerpt: Title: Madagascar's Government Dissolution Amidst Gen Z-Inspired Protests: A Call for Dialogue and Reform Date: 2025-09-29 Source: Devdiscourse  In response to youth-led protests over worsening water and power shortages, Malagasy President Andry Rajoelina announced the dissolution of the government on Monday. The unrest, largely influenced by Gen Z movements in Kenya and Nepal, marks the largest such demonstrations in Madagascar in years. These rallies significantly challenge Rajoelina's leadership since his recent 2023 re-election. The president offered an apology for governmental shortcomings and vowed to engage in dialogue with the youth while ensuring support for affected businesses. The protests have seen significant casualties, with both protestors and bystanders affected, although official figures remain contested. (With inputs from agencies.)...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #2 -------------------------\n",
+      "📅 Date:   2025-10-27\n",
+      "📰 Source: capitalfm\n",
+      "📝 Excerpt: Title: Motorists protest ‘hidden’ concession in Rironi-Mau Summit Expressway Date: 2025-10-27 Source: capitalfm...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #3 -------------------------\n",
+      "📅 Date:   2024-06-21\n",
+      "📰 Source: riotimesonline.com\n",
+      "📝 Excerpt: In Kenya, a significant backlash has emerged against the 2024 Finance Bill, which proposes various tax increases. The bill was approved by the Kenyan parliament. This happened despite modifications by Parliament’s Finance Committee aimed at mitigating public dissatisfaction by dropping several contentious tax proposals. This legislative move coincided with a wave of protests primarily driven by Gen Z and millennials, marking a pivotal moment in the country’s political landscape. Protests erupted across multiple Kenyan cities, including Nairobi and key areas within President William Ruto’s Rift Valley strongholds like Eldoret and Kericho. These demonstrations were notable not only for their scope but also for their organization. They were orchestrated online without the backing of established political parties. The phrase “Tuko wengi,” meaning “We are many,” echoed through the streets of Eldoret, symbolizing the protesters’ unity and significant numbers. The demographic profile of the protesters is particularly noteworthy. Approximately 75% of Kenya’s population is under 35, with a median age of 19. Young Kenyans Mobilize This young populace could be a transformative force in future electoral processes, potentially starting with the 2027 general elections. Their active engagement in these protests reflects a broader trend of increasing political mobilization through digital platforms. This trend facilitates widespread participation and amplifies their collective voice. This series of protests underscores a deep-seated discontent among the youth, who perceive the tax hikes as detrimental to their economic prospects. Their willingness to publicly express their dissatisfaction highlights a shift towards more grassroots political involvement. This suggests that these young, digitally-savvy generations could significantly shape the future of Kenyan politics...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #4 -------------------------\n",
+      "📅 Date:   2023-07-19\n",
+      "📰 Source: abcnews.go.com\n",
+      "📝 Excerpt: . Luis Tato/AFP via Getty Images At least five protesters were injured on Wednesday as demonstrators clashed with police. Amnesty International Kenya said, said that \"para-military police officers and armored water cannon trucks [are] already patrolling and engaging protestors across several towns and neighborhoods.\" In Kibera -- a stronghold of the opposition -- protests turned violent, with demonstrators setting fire to tires and furniture, stones being pelted, and tear gas being deployed by police. In the most recent round of anti-government protests at least 23 people are reported to have been killed according to the U.N., with over 300 arrested. Protests have also been reported in Kenya's Kisumu, Kisii and Migori counties. A Kenya Police Officer shoots a tear gas canister to disperse some protesters as they gather to demonstrate in Nairobi, Kenya, on July 12, 2023. Luis Tato/AFP via Getty Images Kenya's Ministry of Education also announced that all primary and secondary schools in Nairobi and the coastal city Mombasa are to close on Wednesday as a \"precautionary measure\" following \"credible security intelligence.\" Several businesses also remain closed. The protests come after Ruto last month signed into law a contentious finance bill at Nairobi's State House that proposed doubling the tax levied on fuel from 8% to 16%. The bill aimed to aid in offsetting Kenya's external debt, officials said. However, the bill will have a ripple effect on the price of basic commodities, compounding on the economic strain of Kenyans already struggling with the rising cost of living. Riot police detain a supporter of Kenya's opposition leader Raila Odinga as he participates in an anti-government protest against the imposition of tax hikes by the government in Nairobi, Kenya, July 19, 2023. Thomas Mukoya/Reuters Implementation of the Bill -- which was due to come into effect on July 1 -- was halted by Kenya's High Court following a case brought by opposition Sen...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #5 -------------------------\n",
+      "📅 Date:   2023-07-19\n",
+      "📰 Source: abcnews.go.com\n",
+      "📝 Excerpt: An opposition leader called for three days of protests against a finance bill. 6 dead as Kenya rocked by nationwide anti-government protests over gas tax, Amnesty says A riot policeman reloads a teargas grenade launcher during clashes with protesters in the Kibera area of Nairobi, Kenya, July 19, 2023. A riot policeman reloads a teargas grenade launcher during clashes with protesters in the Kibera area of Nairobi, Kenya, July 19, 2023. A riot policeman reloads a teargas grenade launcher during clashes with protesters in the Kibera area of Nairobi, Kenya, July 19, 2023. A riot policeman reloads a teargas grenade launcher during clashes with protesters in the Kibera area of Nairobi, Kenya, July 19, 2023. LONDON -- Kenya was bracing for days of anti-government protests led by the government's political opposition over a contentious new finance bill and the rising cost of living At least six people were shot and killed and at least a dozen others were injured on Wednesday, the first day of a planned three-day protest against higher taxes, Mathias Kinyoda, of Amnesty International Kenya, told ABC News. At least 87 demonstrators were arrested nationwide, he said. The protests were called by opposition leader Raila Odinga. The unrest was set to take place despite Kenya's President William Ruto vowing no protests would take place in the East African Nation. \"We are here, first and foremost, to confirm that the peaceful protests planned for Wednesday, Thursday and Friday this week are on as earlier declared by our leadership,\" read a statement by Odinga's party, Azimio La Umoja, sent to ABC News. A Kenya Police Officer runs away from a group of opposition supporters chasing him and throwing stones during anti-government protests in Nairobi on July 19, 2023. Luis Tato/AFP via Getty Images At least five protesters were injured on Wednesday as demonstrators clashed with police...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #6 -------------------------\n",
+      "📅 Date:   2022-07-20\n",
+      "📰 Source: thesouthafrican.com\n",
+      "📝 Excerpt: Inflation in Kenya: why and how to fix it; Image: Adobe stock Inflation has hit many countries recently, from the United States to Sri Lanka. In Kenya, too, the rising prices of basic commodities have left most citizens wondering what’s going on. The price of a 2kg packet of maize and wheat flour hit 200 shillings (US$2) from a low of 120 shillings in about three months. That is a 67% increase. The 12-month overall inflation rate reached 7.91% in June 2022. Politicians eyeing Kenya’s 9 August polls have been offering solutions in exchange for votes. ADVERTISEMENT Kenya’s average annual per capita income is US$5 270. With inflation, citizens lose even this limited purchasing power. The same money buys less. Wages and salaries do not go up fast enough. Citizens’ discontent can change the way they vote in democratic countries or lead to violence in undemocratic ones. The Kenya African National Union, which ruled the country from independence in 1963, was voted out in 2002 partly because of citizens’ discontent over the state of the economy. And in the US, economic discontent has been a big factor in voting; it led to Donald Trump’s win in the 2016 presidential polls. That’s why politicians are so quick to promise relief. But can they provide it? The two key drivers of inflation in Kenya’s consumer price index are food and energy. Russia’s war on Ukraine has raised the price of oil to the highest level in history, which spills over to the rest of global economy. And about 30% to 50% of Kenya’s imported wheat comes from Russia and Ukraine. Ukraine is exporting 60% less wheat this year compared with 2021, leading to a rise in price of wheat and its derivatives like bread. Unreliable rains have cut domestic production of maize and other food crops in Kenya. Production is about 15% to 20% below the five-year average. ADVERTISEMENT Some of the drivers of Kenya’s inflation are local while others are external and beyond its control...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #7 -------------------------\n",
+      "📅 Date:   2025-10-21\n",
+      "📰 Source: Crypto News\n",
+      "📝 Excerpt: . While Tether hasn’t broken down that figure by region, its gaze is fixed on Africa, where it sees its next chapter of growth unfolding. The firm points to a Chainalysis report revealing a 52% explosion in on-chain transaction volume across Sub-Saharan Africa, which rocketed past $205 billion in a single year. Behind that surge are small business owners and individuals turning to digital assets as a lifeline. They’re navigating the same harsh realities the data confirms: soaring inflation, unpredictable local currencies, and banking systems that have left many behind. To put faces to these numbers, Tether released a short documentary from Kenya. The film highlights local merchants using USDT to pay international suppliers and families relying on it to receive remittances from abroad. It’s a grassroots look at how a global digital dollar is providing a tangible anchor in economies often defined by their volatility....\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #8 -------------------------\n",
+      "📅 Date:   2022-07-20\n",
+      "📰 Source: thesouthafrican.com\n",
+      "📝 Excerpt: . Production is about 15% to 20% below the five-year average. ADVERTISEMENT Some of the drivers of Kenya’s inflation are local while others are external and beyond its control. My view, based on my analysis of the Kenyan economy and other countries, is that inflation can be managed but there are no quick fixes. ALSO READ: Throwback: Family turns ‘white’ after freak accident (pics) Drivers of inflation In Kenya, a confluence of many factors has inflated prices, particularly after the Ukraine war and the pandemic. One is elections. Lots of money is spent during electioneering. Some of it is just given out with no commensurate productivity. Kenya saw this in 1990, when money in circulation rose before the 1992 elections and so did the rate of inflation. The second factor is corruption and mismanagement. Whether it’s in procurement where prices are inflated, or when goods are not supplied or substandard ones are supplied, consumers pay the price. The cost of corruption to the economy has been put by President Uhuru Kenyatta at 2 billion shillings, translating to about 7% of GDP annually. If people make illegal water or power connections, honest people pay for that. If a tender for building a road is inflated, someone pays for it. If government and its agencies over-employ, someone pays for that. If it takes longer to get services like port clearance and building approvals, someone pays for it. If bribes are exchanged, someone pays for it. The 50-shilling note given to police at a roadblock is paid by someone else. A third driver of inflation is a weak currency. Kenya’s currency has declined by 3.5% since the start of the year, partly because of decisions taken in other countries that affect the value of their currencies. A fourth driver of inflation is tax. It raises the price of goods and services. The Finance Act 2022 brought in new taxes and raised the rates of other existing ones. It seems the government did this to raise money without incurring more debts...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #9 -------------------------\n",
+      "📅 Date:   2023-06-23\n",
+      "📰 Source: france24.com\n",
+      "📝 Excerpt: Title: Kenya: Tax on all salaries to finance housing Date: 2023-06-23 Source: france24.com  Kenya's parliament's voted throuh a raft of tax hikes in the first budget of president William Ruto. The contentious changes will double the tax on fuel to 16% to generate another 61 billion shillings or a little under 400,000 euros, for the government. The bill still has to be signed by Ruto and opposition members have threatened to call fresh protests if he does. The bill also proposes a new housing levy which Ruto says will help build more affordable houses but many Kenyans hit hard by the rising cost of living say they can't shoulder the extra cost. FRANCE 24's correspondent Bastien Renouil reports. Video by: Bastien RENOUIL...\n",
+      "------------------------------------\n",
+      "\n",
+      "Result #10 -------------------------\n",
+      "📅 Date:   2025-11-10\n",
+      "📰 Source: capitalfm\n",
+      "📝 Excerpt: NAIROBI, Kenya, Nov 10 — President William Ruto says he has “no regrets” over his directive to police to shoot protesters “in the feet,” defending the use of force during violent anti-government demonstrations last year. Speaking in an interview with Al Jazeera aired on Sunday, Ruto maintained that police acted lawfully when responding to riots that left dozens dead and businesses destroyed. “I don’t regret those comments at all because the law allows the police to use force when other people’s lives are in danger,” he said. When challenged on whether shooting protesters was excessive, he replied, “That is according to you. I think the police know what they need to do.” The President said his administration had to “balance” between allowing peaceful demonstrations and curbing criminal activity during the unrest. “We have had to balance between dealing with violent criminals on one end and managing protests on the other,” he told James Bays. Ruto says police to immobilize vandals, take them to court » Capital News July 9 directive Ruto’s comments referred to a July 9 address in Nairobi’s Kilimani area, where he warned vandals and looters that they would be immobilized and taken to court. “Anybody torching and destroying another person’s business should be shot in the feet and taken to the hospital pending court appearance,” Ruto declared at the time. “We want people to do business. Enough is enough.” The President spoke amid escalating anti-government protests that saw supermarkets and small businesses looted and torched in several towns, including Meru, Kitengela, and Kahawa Sukari. Magunas Supermarket in Meru was among the worst hit — ransacked and later set ablaze. According to the Kenya National Commission on Human Rights (KNCHR), thirty-one people were killed in the early wave of protests, with the toll later rising to sixty-five as demonstrations continued through June and July...\n",
+      "------------------------------------\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# --- TEST QUERY ---\n",
+    "query = \"How have the protests impacted the Kenyan economy?\"\n",
+    "\n",
+    "print(f\"\\n🔎 Searching for: '{query}'...\")\n",
+    "\n",
+    "# Perform Similarity Search\n",
+    "results = vectorstore.similarity_search(query, k=10)\n",
+    "\n",
+    "print(f\"Found {len(results)} relevant articles:\\n\")\n",
+    "\n",
+    "for i, doc in enumerate(results):\n",
+    "    print(f\"Result #{i+1} -------------------------\")\n",
+    "    print(f\"Date:   {doc.metadata.get('date', 'N/A')}\")\n",
+    "    print(f\"Source: {doc.metadata.get('source', 'N/A')}\")\n",
+    "    print(f\"Excerpt: {doc.page_content[:2500].replace(chr(10), ' ')}...\") # Remove newlines for clean print\n",
+    "    print(\"------------------------------------\\n\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44fc797c-1462-48a6-9eba-60e06afbd6c9",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/test_demo.ipynb

@@ -0,0 +1,309 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d5c236df-3b17-4889-bfa0-62875afabb70",
+   "metadata": {},
+   "source": [
+    "# Demo notebook"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "ae890231-b53d-451b-912c-ad84bd1f3360",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install langchain-ollama --quiet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "ed3dafc1-5641-42ee-94e4-299295939a8f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "25b50f89-d265-42b1-a97b-8e7790856595",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fix paths for src files\n",
+    "project_root = Path(os.getcwd()).parent\n",
+    "script_dir = project_root / \"src\"\n",
+    "if str(script_dir) not in sys.path:\n",
+    "    sys.path.append(str(script_dir))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "dea362be-192d-4929-b140-3778cba1df25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import warnings\n",
+    "warnings.filterwarnings(\"ignore\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "333e0906-23f5-4836-8beb-188d2155e879",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.mshauri_demo import create_mshauri_agent, ask_mshauri"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "d8f69073-a21a-4b9d-89c5-d7554d7ac605",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⚙️  Initializing Mshauri Fedha (Model: qwen3:32b)...\n",
+      "✅ Mshauri Agent Ready (Zero-Dependency Mode).\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Initialize agent\n",
+    "agent = create_mshauri_agent()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "7a607e94-d378-4127-9c97-1d0a5a3e629f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query1 = \"What was the total value of commodity tea exports in 1998?\"\n",
+    "query2 = \"Why has the cost of living increased according to the reports?\"\n",
+    "query3 = \"What is the latest inflation rate?\"\n",
+    "query4 = \"What is the annual GDP for 2020?\"\n",
+    "query5 = \"How is the Kenyan economy performing compared to other African countries and countries like USA and Australia?\"\n",
+    "query6 = \"What sector of the Kenyan economy has been constantly improving? Show the numbers\"\n",
+    "query7 = \"summarize recent loan default trends in microfinance institutions.\"\n",
+    "query8 = \"What was the total public debt for 1999?\"\n",
+    "query9 = \"Is the Kenyan economy improving? Considering the quality of life of its citizens\"\n",
+    "query10 = \"Why did the shilling depreciate?\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "5d5a7ff3-aa69-4083-81cf-d6e34c41be6f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from load.start_ollama import start_ollama_server"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "dc20cc1d-da64-4394-85d5-7b77ad0e88c5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🚀 Starting Ollama Server...\n",
+      "⏳ Waiting for server to boot...\n",
+      "✅ Server started successfully.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Start ollama server\n",
+    "start_ollama_server()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e3942375-e589-4469-94ef-b0bae5e0684f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "❓ User: Why did the shilling depreciate?\n",
+      "----------------------------------------\n",
+      "🚀 Starting Agent Loop for: 'Why did the shilling depreciate?'\n",
+      "\n",
+      "🧠 Step 1: Thought: I need to determine the reasons behind the Kenyan shilling's depreciation. This requires analyzing economic factors like inflation, trade deficits, political stability, or global market trends. I'll first check the database for relevant quantitative data, then use financial reports/news if needed.\n",
+      "\n",
+      "Action: sql_db_list_tables  \n",
+      "Action Input:\n",
+      "🛠️  Calling 'sql_db_list_tables' with: \n",
+      "\n",
+      "🧠 Step 2: Thought: I need to analyze economic indicators like trade deficits, inflation, or central bank policies. First, check the foreign trade summary for import/export trends and exchange rate history.\n",
+      "\n",
+      "Action: sql_db_schema  \n",
+      "Action Input: foreign_trade_summary, cbk_indicative_rates\n",
+      "🛠️  Calling 'sql_db_schema' with: foreign_trade_summary, cbk_indicative_rates\n",
+      "\n",
+      "🧠 Step 3: Thought: I need to analyze trade deficits and exchange rate trends. First, check recent trade balances and exchange rates.\n",
+      "\n",
+      "Action: sql_db_query_checker  \n",
+      "Action Input: SELECT year, month, trade_balance FROM foreign_trade_summary ORDER BY year DESC, month DESC LIMIT 12\n",
+      "🛠️  Calling 'sql_db_query_checker' with: SELECT year, month, trade_balance FROM foreign_trade_summary ORDER BY year DESC, month DESC LIMIT 12\n",
+      "\n",
+      "🧠 Step 4: Action: sql_db_query  \n",
+      "Action Input: SELECT year, month, trade_balance FROM foreign_trade_summary ORDER BY year DESC, month DESC LIMIT 12\n",
+      "🛠️  Calling 'sql_db_query' with: SELECT year, month, trade_balance FROM foreign_trade_summary ORDER BY year DESC, month DESC LIMIT 12\n",
+      "\n",
+      "🧠 Step 5: Thought: The trade balance data shows a consistent deficit, indicating higher imports than exports, which increases demand for foreign currency. This likely contributes to shilling depreciation. To confirm, check exchange rate trends over time.\n",
+      "\n",
+      "Action: sql_db_query_checker  \n",
+      "Action Input: SELECT date, mean_rate FROM cbk_indicative_rates WHERE currency = 'US DOLLAR' ORDER BY date DESC LIMIT 12\n",
+      "🛠️  Calling 'sql_db_query_checker' with: SELECT date, mean_rate FROM cbk_indicative_rates WHERE currency = 'US DOLLAR' ORDER BY date DESC LIMIT 12\n",
+      "\n",
+      "🧠 Step 6: Thought: The trade balance data shows persistent deficits, increasing demand for foreign currency. Exchange rate trends for the US dollar (mean rate rising) confirm shilling depreciation. To fully answer \"why,\" I should also check for inflation data or policy changes, but since the database lacks an inflation table, I'll rely on available data and external sources.\n",
+      "\n",
+      "Action: search_financial_reports_and_news  \n",
+      "Action Input: Kenyan shilling depreciation causes 2024-2025\n",
+      "🛠️  Calling 'search_financial_reports_and_news' with: Kenyan shilling depreciation causes 2024-2025\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/users/tligawa/mshauri-fedha/notebooks/mshauri_demo.py:159: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the `langchain-ollama package and should be used instead. To use it run `pip install -U `langchain-ollama` and import as `from `langchain_ollama import OllamaEmbeddings``.\n",
+      "  embeddings = OllamaEmbeddings(model=DEFAULT_EMBED_MODEL, base_url=ollama_url)\n",
+      "/users/tligawa/mshauri-fedha/notebooks/mshauri_demo.py:160: LangChainDeprecationWarning: The class `Chroma` was deprecated in LangChain 0.2.9 and will be removed in 1.0. An updated version of the class exists in the `langchain-chroma package and should be used instead. To use it run `pip install -U `langchain-chroma` and import as `from `langchain_chroma import Chroma``.\n",
+      "  vectorstore = Chroma(persist_directory=vector_db_path, embedding_function=embeddings)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "🧠 Step 7: The depreciation of the Kenyan shilling (KES) can be attributed to a combination of economic factors, supported by data from trade, monetary policy, and sectoral performance:\n",
+      "\n",
+      "### 1. **Persistent Trade Deficits**  \n",
+      "   - **Foreign Trade Data**: The *foreign_trade_summary* table reveals consistent negative trade balances (e.g., large deficits in 2022–2023). Kenya imports significantly more goods (e.g., machinery, fuel, and manufactured products) than it exports, increasing demand for foreign currency and weakening the shilling.  \n",
+      "   - **Impact**: A trade deficit reduces foreign exchange reserves, making the currency more vulnerable to depreciation.\n",
+      "\n",
+      "### 2. **Low Central Bank Intervention**  \n",
+      "   - **CBK Policy**: News articles highlight the Central Bank of Kenya (CBK) has limited interventions in the foreign exchange market. Without active measures to stabilize the shilling (e.g., selling foreign reserves), market forces drive the currency lower.  \n",
+      "   - **Market Sentiment**: Analysts predict the shilling could fall below **KES 90 to the dollar** by year-end, reflecting weak confidence in sustained CBK support.\n",
+      "\n",
+      "### 3. **Low Interest Rates**  \n",
+      "   - **Monetary Policy (Table 16)**: Savings deposit rates (e.g., 3.56% in 2022) and inter-bank rates (5.39% in 2022) are historically low, reducing incentives for foreign investors to hold shillings. This drives capital outflows, further pressuring the currency.  \n",
+      "   - **Comparison**: Overdraft and loan rates (12–12.67%) remain high for borrowers but fail to attract external investment, as returns in shillings are eroded by depreciation.\n",
+      "\n",
+      "### 4. **Weak Export Sector Growth**  \n",
+      "   - **Agriculture Data (Table 33)**: While tea and pyrethrum production has grown, key exports like maize and wheat show volatility. Agriculture contributes ~21% of GDP but lacks diversification or value addition to boost export earnings.  \n",
+      "   - **Non-Agricultural Exports**: Limited growth in manufacturing and services (e.g., tourism) exacerbates the trade deficit, reducing foreign exchange inflows.\n",
+      "\n",
+      "### 5. **Global and Regional Factors**  \n",
+      "   - **Commodity Prices**: Higher global fuel and food prices increase import costs, worsening the trade deficit.  \n",
+      "   - **Regional Currency Dynamics**: The shilling’s depreciation relative to neighbors (e.g., Uganda, Tanzania) reflects broader East African economic challenges, including inflation and weak regional trade integration.\n",
+      "\n",
+      "### Conclusion  \n",
+      "The shilling’s depreciation stems from structural issues: **trade imbalances**, **limited CBK intervention**, **low interest rates**, and **stagnant export growth**. Without addressing these fundamentals (e.g., boosting exports, attracting foreign investment, or stabilizing the exchange rate), the shilling is likely to remain under pressure. Policymakers must balance short-term market interventions with long-term structural reforms to restore currency stability.\n",
+      "----------------------------------------\n",
+      "💡 Mshauri: The depreciation of the Kenyan shilling (KES) can be attributed to a combination of economic factors, supported by data from trade, monetary policy, and sectoral performance:\n",
+      "\n",
+      "### 1. **Persistent Trade Deficits**  \n",
+      "   - **Foreign Trade Data**: The *foreign_trade_summary* table reveals consistent negative trade balances (e.g., large deficits in 2022–2023). Kenya imports significantly more goods (e.g., machinery, fuel, and manufactured products) than it exports, increasing demand for foreign currency and weakening the shilling.  \n",
+      "   - **Impact**: A trade deficit reduces foreign exchange reserves, making the currency more vulnerable to depreciation.\n",
+      "\n",
+      "### 2. **Low Central Bank Intervention**  \n",
+      "   - **CBK Policy**: News articles highlight the Central Bank of Kenya (CBK) has limited interventions in the foreign exchange market. Without active measures to stabilize the shilling (e.g., selling foreign reserves), market forces drive the currency lower.  \n",
+      "   - **Market Sentiment**: Analysts predict the shilling could fall below **KES 90 to the dollar** by year-end, reflecting weak confidence in sustained CBK support.\n",
+      "\n",
+      "### 3. **Low Interest Rates**  \n",
+      "   - **Monetary Policy (Table 16)**: Savings deposit rates (e.g., 3.56% in 2022) and inter-bank rates (5.39% in 2022) are historically low, reducing incentives for foreign investors to hold shillings. This drives capital outflows, further pressuring the currency.  \n",
+      "   - **Comparison**: Overdraft and loan rates (12–12.67%) remain high for borrowers but fail to attract external investment, as returns in shillings are eroded by depreciation.\n",
+      "\n",
+      "### 4. **Weak Export Sector Growth**  \n",
+      "   - **Agriculture Data (Table 33)**: While tea and pyrethrum production has grown, key exports like maize and wheat show volatility. Agriculture contributes ~21% of GDP but lacks diversification or value addition to boost export earnings.  \n",
+      "   - **Non-Agricultural Exports**: Limited growth in manufacturing and services (e.g., tourism) exacerbates the trade deficit, reducing foreign exchange inflows.\n",
+      "\n",
+      "### 5. **Global and Regional Factors**  \n",
+      "   - **Commodity Prices**: Higher global fuel and food prices increase import costs, worsening the trade deficit.  \n",
+      "   - **Regional Currency Dynamics**: The shilling’s depreciation relative to neighbors (e.g., Uganda, Tanzania) reflects broader East African economic challenges, including inflation and weak regional trade integration.\n",
+      "\n",
+      "### Conclusion  \n",
+      "The shilling’s depreciation stems from structural issues: **trade imbalances**, **limited CBK intervention**, **low interest rates**, and **stagnant export growth**. Without addressing these fundamentals (e.g., boosting exports, attracting foreign investment, or stabilizing the exchange rate), the shilling is likely to remain under pressure. Policymakers must balance short-term market interventions with long-term structural reforms to restore currency stability.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'The depreciation of the Kenyan shilling (KES) can be attributed to a combination of economic factors, supported by data from trade, monetary policy, and sectoral performance:\\n\\n### 1. **Persistent Trade Deficits**  \\n   - **Foreign Trade Data**: The *foreign_trade_summary* table reveals consistent negative trade balances (e.g., large deficits in 2022–2023). Kenya imports significantly more goods (e.g., machinery, fuel, and manufactured products) than it exports, increasing demand for foreign currency and weakening the shilling.  \\n   - **Impact**: A trade deficit reduces foreign exchange reserves, making the currency more vulnerable to depreciation.\\n\\n### 2. **Low Central Bank Intervention**  \\n   - **CBK Policy**: News articles highlight the Central Bank of Kenya (CBK) has limited interventions in the foreign exchange market. Without active measures to stabilize the shilling (e.g., selling foreign reserves), market forces drive the currency lower.  \\n   - **Market Sentiment**: Analysts predict the shilling could fall below **KES 90 to the dollar** by year-end, reflecting weak confidence in sustained CBK support.\\n\\n### 3. **Low Interest Rates**  \\n   - **Monetary Policy (Table 16)**: Savings deposit rates (e.g., 3.56% in 2022) and inter-bank rates (5.39% in 2022) are historically low, reducing incentives for foreign investors to hold shillings. This drives capital outflows, further pressuring the currency.  \\n   - **Comparison**: Overdraft and loan rates (12–12.67%) remain high for borrowers but fail to attract external investment, as returns in shillings are eroded by depreciation.\\n\\n### 4. **Weak Export Sector Growth**  \\n   - **Agriculture Data (Table 33)**: While tea and pyrethrum production has grown, key exports like maize and wheat show volatility. Agriculture contributes ~21% of GDP but lacks diversification or value addition to boost export earnings.  \\n   - **Non-Agricultural Exports**: Limited growth in manufacturing and services (e.g., tourism) exacerbates the trade deficit, reducing foreign exchange inflows.\\n\\n### 5. **Global and Regional Factors**  \\n   - **Commodity Prices**: Higher global fuel and food prices increase import costs, worsening the trade deficit.  \\n   - **Regional Currency Dynamics**: The shilling’s depreciation relative to neighbors (e.g., Uganda, Tanzania) reflects broader East African economic challenges, including inflation and weak regional trade integration.\\n\\n### Conclusion  \\nThe shilling’s depreciation stems from structural issues: **trade imbalances**, **limited CBK intervention**, **low interest rates**, and **stagnant export growth**. Without addressing these fundamentals (e.g., boosting exports, attracting foreign investment, or stabilizing the exchange rate), the shilling is likely to remain under pressure. Policymakers must balance short-term market interventions with long-term structural reforms to restore currency stability.'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ask_mshauri(agent, query10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "79f0ba2b-7f35-4a2c-80fe-4d9c3731c558",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/transform.ipynb

@@ -0,0 +1,315 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "84e9f72e-84ff-49e5-b8ba-faa6ee9bc4df",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2b495825-0d1a-4a46-9297-6ceae1ccd2a2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "import shutil\n",
+    "import subprocess\n",
+    "import time\n",
+    "import requests\n",
+    "import torch\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Fix paths so we can import 'extract.py'\n",
+    "project_root = Path(os.getcwd()).parent\n",
+    "script_dir = project_root / \"src/transform\"\n",
+    "if str(script_dir) not in sys.path:\n",
+    "    sys.path.append(str(script_dir))\n",
+    "\n",
+    "# Import your optimized processor\n",
+    "from extract import MarkerFolderProcessor, configure_parallelism"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "8d04e7ad-abf2-40e4-b308-fc0863464935",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Setup complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Paths\n",
+    "SCRATCH = Path(os.environ.get(\"SCRATCH\"))\n",
+    "INPUT_PDFS = SCRATCH / \"mshauri-fedha/data/cbk/pdfs\"\n",
+    "OUTPUT_DIR = SCRATCH / \"mshauri-fedha/data/cbk/marker-output\"\n",
+    "\n",
+    "# Ollama Setup\n",
+    "OLLAMA_HOME = SCRATCH / \"ollama_core\"\n",
+    "OLLAMA_BIN = OLLAMA_HOME / \"bin/ollama\"\n",
+    "OLLAMA_MODELS_DIR = OLLAMA_HOME / \"models\" \n",
+    "OLLAMA_HOST = \"http://localhost:11434\"\n",
+    "\n",
+    "print(\"✅ Setup complete.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2a7846b4-2041-4b4f-9210-16a891d6c9f4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🔍 GH200/A100 Detected: 4 GPUs | 94.5 GB VRAM\n",
+      "⚙️  Stability Config: 5 workers/GPU | 20 Total Slots\n"
+     ]
+    }
+   ],
+   "source": [
+    "total_slots, workers_per_gpu, num_gpus = configure_parallelism()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "039a0a95-91e2-495c-a0ab-d2185f98461c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Kill any old server first\n",
+    "subprocess.run([\"pkill\", \"-f\", \"ollama serve\"], stderr=subprocess.DEVNULL)\n",
+    "time.sleep(2)\n",
+    "\n",
+    "server_env = os.environ.copy()\n",
+    "server_env[\"OLLAMA_NUM_PARALLEL\"] = str(32) # Matches your total slots\n",
+    "server_env[\"OLLAMA_MAX_LOADED_MODELS\"] = \"1\"\n",
+    "server_env[\"OLLAMA_MAX_QUEUE\"] = \"2048\"\n",
+    "\n",
+    "# Start new server\n",
+    "process = subprocess.Popen(\n",
+    "    [str(OLLAMA_BIN), \"serve\"], \n",
+    "    stdout=subprocess.DEVNULL, \n",
+    "    stderr=subprocess.DEVNULL,\n",
+    "    env=server_env\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "accd6a19-e216-450e-9cca-beaeaa7749a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⏳ Waiting for server heartbeat...\n",
+      "✅ Server is UP and listening!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Robust Wait Loop\n",
+    "print(\"⏳ Waiting for server heartbeat...\")\n",
+    "server_ready = False\n",
+    "for _ in range(60): # Wait 60 seconds max\n",
+    "    try:\n",
+    "        if requests.get(OLLAMA_HOST).status_code == 200:\n",
+    "            server_ready = True\n",
+    "            break\n",
+    "    except:\n",
+    "        time.sleep(1)\n",
+    "\n",
+    "if server_ready:\n",
+    "    print(\"✅ Server is UP and listening!\")\n",
+    "else:\n",
+    "    raise RuntimeError(\"❌ Server failed to start. Check logs.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "454617a5-c1ef-489f-b9c0-8e6b4fe39b47",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "⬇️  Checking/Pulling qwen2.5:7b...\n",
+      "📝 Creating 'qwen2.5-7b-16k' (16k Context)...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# pull model\n",
+    "BASE_MODEL = \"qwen2.5:7b\" \n",
+    "CUSTOM_MODEL_NAME = \"qwen2.5-7b-16k\"\n",
+    "\n",
+    "print(f\"⬇️  Checking/Pulling {BASE_MODEL}...\")\n",
+    "subprocess.run(\n",
+    "    [str(OLLAMA_BIN), \"pull\", BASE_MODEL], \n",
+    "    check=True, \n",
+    "    stdout=subprocess.DEVNULL,\n",
+    "    stderr=subprocess.DEVNULL,\n",
+    "    env=os.environ.copy()\n",
+    ")\n",
+    "\n",
+    "print(f\"📝 Creating '{CUSTOM_MODEL_NAME}' (16k Context)...\")\n",
+    "modelfile_content = f\"FROM {BASE_MODEL}\\nPARAMETER num_ctx 16384\"\n",
+    "with open(\"Modelfile_qwen_16k\", \"w\") as f:\n",
+    "    f.write(modelfile_content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "0fe26ed7-f31f-43eb-acca-1795b5528219",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model Ready.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[?2026h\u001b[?25l\u001b[1Ggathering model components \u001b[K\n",
+      "using existing layer sha256:2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730 \u001b[K\n",
+      "using existing layer sha256:66b9ea09bd5b7099cbb4fc820f31b575c0366fa439b08245566692c6784e281e \u001b[K\n",
+      "using existing layer sha256:eb4402837c7829a690fa845de4d7f3fd842c2adee476d5341da8a46ea9255175 \u001b[K\n",
+      "using existing layer sha256:832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e \u001b[K\n",
+      "using existing layer sha256:db8fbfd0cb288a053f83ac9014ca9bac2558b1bbcd80b5c408a548e7acba8a24 \u001b[K\n",
+      "writing manifest ⠋ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[1Ggathering model components \u001b[K\n",
+      "using existing layer sha256:2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730 \u001b[K\n",
+      "using existing layer sha256:66b9ea09bd5b7099cbb4fc820f31b575c0366fa439b08245566692c6784e281e \u001b[K\n",
+      "using existing layer sha256:eb4402837c7829a690fa845de4d7f3fd842c2adee476d5341da8a46ea9255175 \u001b[K\n",
+      "using existing layer sha256:832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e \u001b[K\n",
+      "using existing layer sha256:db8fbfd0cb288a053f83ac9014ca9bac2558b1bbcd80b5c408a548e7acba8a24 \u001b[K\n",
+      "writing manifest \u001b[K\n",
+      "success \u001b[K\u001b[?25h\u001b[?2026l\n"
+     ]
+    }
+   ],
+   "source": [
+    "# run model\n",
+    "subprocess.run(\n",
+    "    [str(OLLAMA_BIN), \"create\", CUSTOM_MODEL_NAME, \"-f\", \"Modelfile_qwen\"], \n",
+    "    check=True, \n",
+    "    stdout=subprocess.DEVNULL, \n",
+    "    env=os.environ.copy()\n",
+    ")\n",
+    "print(\"✅ Model Ready.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "4750bd0f-3cd2-4d62-a6c4-75c2f19e45f1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "os.chdir(SCRATCH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9581ee47-f690-46c8-b331-084411fb8535",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Detected 4 GPUs (Dynamic Mode)\n",
+      "🚀 Processing PDFs from: /capstor/scratch/cscs/tligawa/mshauri-fedha/data/cbk/pdfs\n",
+      "📦 Created 1089 batches of 5 files each.\n",
+      "🚀 Launching 20 workers on 4 GPUs...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "20:26:57 - [GPU-3:Dev3] - Initializing Worker 3...\n",
+      "20:26:58 - [GPU-0:Dev0] - Initializing Worker 0...\n",
+      "20:27:05 - [GPU-1:Dev1] - Initializing Worker 1...\n",
+      "20:27:06 - [GPU-2:Dev2] - Initializing Worker 2...\n",
+      "20:27:09 - [GPU-4:Dev0] - Initializing Worker 4...\n",
+      "20:27:11 - [GPU-5:Dev1] - Initializing Worker 5...\n",
+      "20:27:12 - [GPU-6:Dev2] - Initializing Worker 6...\n",
+      "20:27:12 - [GPU-9:Dev1] - Initializing Worker 9...\n",
+      "20:27:14 - [GPU-7:Dev3] - Initializing Worker 7...\n",
+      "20:27:15 - [GPU-8:Dev0] - Initializing Worker 8...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Initialize the Processor\n",
+    "processor = MarkerFolderProcessor(\n",
+    "    output_dir=OUTPUT_DIR,\n",
+    "    ollama_url=OLLAMA_HOST,\n",
+    "    ollama_model=CUSTOM_MODEL_NAME,\n",
+    "    batch_multiplier=4,                  \n",
+    "    workers_per_gpu=workers_per_gpu,\n",
+    "    num_gpus=num_gpus                   \n",
+    ")\n",
+    "\n",
+    "# 3. Run the extraction\n",
+    "print(f\"🚀 Processing PDFs from: {INPUT_PDFS}\")\n",
+    "processor.process_folder(INPUT_PDFS, batch_size=5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "931650d0-50f1-48c1-a1a1-a561392e004b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/unzip_stores.ipynb

@@ -0,0 +1,197 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4a41dc7b-f751-4818-912d-21241047c485",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "729a7545-088d-4521-8948-60162d80b1e7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import shutil\n",
+    "import zipfile\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "96eb6a92-9786-48b6-88e4-0441d1a531c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# --- CONFIGURATION ---\n",
+    "# 1. Source (Permanent Storage)\n",
+    "PROJECT_DIR = Path(os.environ.get(\"PROJECT\"))  # Auto-detects $PROJECT\n",
+    "SOURCE_ZIPS = PROJECT_DIR / \"tligawa/mshauri-fedha-store/cbk/zipped-store\"\n",
+    "\n",
+    "# 2. Destination (Fast Scratch Storage)\n",
+    "SCRATCH_DIR = Path(os.environ.get(\"SCRATCH\"))  # Auto-detects $SCRATCH\n",
+    "WORK_DIR = SCRATCH_DIR / \"mshauri-fedha/data/cbk\"\n",
+    "FINAL_PDF_DIR = WORK_DIR / \"text\"\n",
+    "TEMP_EXTRACT_DIR = WORK_DIR / \"temp-unzip-cbk\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3148a17b-8e99-448e-95de-bb2c60828049",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.path.exists(WORK_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "cc97d707-f266-4f6f-9346-19e630101923",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🚀 Found 111 batches in /capstor/store/cscs/director2/g164/tligawa/mshauri-fedha-store/cbk/zipped-store\n",
+      "📂 Flattening to: /capstor/scratch/cscs/tligawa/mshauri-fedha/data/cbk/text ...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Setup directories\n",
+    "if FINAL_PDF_DIR.exists():\n",
+    "    print(f\"⚠️ Warning: Target folder {FINAL_PDF_DIR} already exists.\")\n",
+    "else:\n",
+    "    FINAL_PDF_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "    \n",
+    "if TEMP_EXTRACT_DIR.exists(): shutil.rmtree(TEMP_EXTRACT_DIR)\n",
+    "TEMP_EXTRACT_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# --- EXECUTION ---\n",
+    "zips = sorted(list(SOURCE_ZIPS.glob(\"*.zip\")))\n",
+    "print(f\"🚀 Found {len(zips)} batches in {SOURCE_ZIPS}\")\n",
+    "print(f\"📂 Flattening to: {FINAL_PDF_DIR} ...\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0c2a9b27-133a-4960-8920-45ee57eb3d8a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Unzipping & Flattening: 100%|██████████| 111/111 [00:22<00:00,  4.84it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "✨ Done! 58 files are ready in /capstor/scratch/cscs/tligawa/mshauri-fedha/data/cbk/text\n",
+      "🧹 Cleaning up temp dirs...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Unzip and flatten\n",
+    "count = 0\n",
+    "for zip_path in tqdm(zips, desc=\"Unzipping & Flattening\"):\n",
+    "    batch_name = zip_path.stem  # e.g., \"knbs_batch_1\"\n",
+    "    \n",
+    "    try:\n",
+    "        # 1. Unzip to a temp folder\n",
+    "        with zipfile.ZipFile(zip_path, 'r') as z:\n",
+    "            z.extractall(TEMP_EXTRACT_DIR)\n",
+    "        \n",
+    "        # 2. Find the 'pdfs' subfolder inside that batch\n",
+    "        # We look recursively because structure might vary slightly\n",
+    "        pdf_files = list(TEMP_EXTRACT_DIR.rglob(\"*.txt\"))\n",
+    "        \n",
+    "        # 3. Move and Rename\n",
+    "        for pdf in pdf_files:\n",
+    "            # Create unique name: batch_name + original_name\n",
+    "            # Example: knbs_batch_1_annual_report_2020.pdf\n",
+    "            new_name = f\"{batch_name}_{pdf.name}\"\n",
+    "            dest_path = FINAL_PDF_DIR / new_name\n",
+    "            \n",
+    "            shutil.move(str(pdf), str(dest_path))\n",
+    "            count += 1\n",
+    "            \n",
+    "    except Exception as e:\n",
+    "        print(f\"❌ Error processing {zip_path.name}: {e}\")\n",
+    "    finally:\n",
+    "        # Clean temp folder for next batch\n",
+    "        for item in TEMP_EXTRACT_DIR.iterdir():\n",
+    "            if item.is_dir(): shutil.rmtree(item)\n",
+    "            else: item.unlink()\n",
+    "\n",
+    "print(f\"\\n✨ Done! {count} files are ready in {FINAL_PDF_DIR}\")\n",
+    "print(f\"🧹 Cleaning up temp dirs...\")\n",
+    "if TEMP_EXTRACT_DIR.exists(): shutil.rmtree(TEMP_EXTRACT_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a04d70da-e9e9-4b1b-a393-93b5b76fcd8b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

src/extract/download_file_links.py

@@ -0,0 +1,671 @@
+# Import dependencies
+from typing import Any, Union, List, Dict
+import time
+import pandas as pd
+import logging
+import requests
+from gnews import GNews
+import feedparser
+from io import BytesIO
+import time, re
+from bs4 import BeautifulSoup
+import urllib3
+import certifi
+from urllib.parse import urljoin, urlparse
+from urllib.robotparser import RobotFileParser
+from collections import Counter
+from tqdm.auto import tqdm
+from newspaper import Article
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+# Set up basic logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class CBKExplorer:
+    def __init__(self, github_username):
+        self.user_agent = f"MshauriFedhaBot/0.1 (+https://github.com/{github_username}/mshaurifedha)"
+        self.session = requests.Session()
+        self.session.headers.update({"User-Agent": self.user_agent})
+
+    def is_allowed_by_robots(self, base_url, target_url):
+        """Check robots.txt for permission."""
+        parsed = urlparse(base_url)
+        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+        rp = RobotFileParser()
+        try:
+            rp.set_url(robots_url)
+            rp.read()
+            return rp.can_fetch(self.user_agent, target_url)
+        except Exception as e:
+            print(f"[robots] Could not read robots.txt ({e}). Proceed cautiously.")
+            return False
+
+    def fetch(self, url, timeout=25, allow_proxy_fallback=True):
+        """
+        Robust fetch that tries:
+        1) requests with certifi bundle (secure)
+        2) http fallback (if https fails)
+        3) requests with verify=False (insecure)
+        4) optional external proxy fetch (r.jina.ai) as last resort
+
+        Returns (response, soup) or (None, None)
+        """
+        # helper to parse response->soup
+        def resp_to_soup(r):
+            try:
+                r.raise_for_status()
+                return r, BeautifulSoup(r.text, "lxml")
+            except Exception:
+                return None, None
+
+        # 1) Try with certifi (preferred)
+        try:
+            r = self.session.get(url, timeout=timeout, verify=certifi.where())
+            ok_resp, soup = resp_to_soup(r)
+            if ok_resp:
+                return ok_resp, soup
+        except requests.exceptions.SSLError as ssl_err:
+            print(f"[fetch] SSL error with certifi for {url}: {ssl_err}")
+        except Exception as e:
+            print(f"[fetch] Primary attempt failed for {url}: {e}")
+
+        # 2) Try http fallback if URL is https
+        try:
+            parsed = urlparse(url)
+            if parsed.scheme == "https":
+                http_url = url.replace("https://", "http://", 1)
+                try:
+                    r = self.session.get(http_url, timeout=timeout)
+                    ok_resp, soup = resp_to_soup(r)
+                    if ok_resp:
+                        print(f"[fetch] HTTP fallback succeeded for {http_url}")
+                        return ok_resp, soup
+                except Exception as e:
+                    print(f"[fetch] HTTP fallback failed for {http_url}: {e}")
+        except Exception as e:
+            print(f"[fetch] HTTP fallback: error preparing URL: {e}")
+
+        # 3) Try insecure (verify=False) as last direct option
+        try:
+            print(f"[fetch] Trying insecure fetch (verify=False) for {url} — not recommended for sensitive data.")
+            r = self.session.get(url, timeout=timeout, verify=False)
+            ok_resp, soup = resp_to_soup(r)
+            if ok_resp:
+                return ok_resp, soup
+        except Exception as e:
+            print(f"[fetch] Insecure fetch also failed for {url}: {e}")
+
+        # 4) Optional: external proxy/relay (last resort)
+        if allow_proxy_fallback:
+            try:
+                # Jina.ai simple fetch service: returns rendered HTML as text
+                # NOTE: this is an external service — use only for public/cached pages.
+                proxy_url = "https://r.jina.ai/http://" + url.replace("https://", "").replace("http://", "")
+                print(f"[fetch] Trying proxy fetch via {proxy_url}")
+                r = requests.get(proxy_url, timeout=30)  # using plain requests (no verify issues; it's https to jina)
+                if r.status_code == 200 and r.text:
+                    return r, BeautifulSoup(r.text, "lxml")
+                else:
+                    print(f"[fetch] Proxy fetch returned status {r.status_code}")
+            except Exception as e:
+                print(f"[fetch] Proxy fetch failed: {e}")
+
+        # give up
+        print(f"[fetch] All fetch strategies failed for {url}")
+        return None, None
+
+    def abs_link(self, base, href):
+        """Make absolute link from relative href."""
+        if not href:
+            return None
+        return urljoin(base, href)
+
+    def explore_url(self, url, print_anchors=40):
+        """Explore a CBK URL: meta, headings, nav links, anchor samples, file-like links."""
+        print("URL:", url)
+        print("Allowed by robots.py? ->", self.is_allowed_by_robots(url, url))
+        resp, soup = self.fetch(url)
+        if not resp:
+            return None
+
+        # Basic meta
+        print("Status code:", resp.status_code)
+        title = soup.title.string.strip() if soup.title else ""
+        print("Title:", title)
+        desc = ""
+        meta_desc = soup.find("meta", attrs={"name":"description"}) or soup.find("meta", attrs={"property":"og:description"})
+        if meta_desc and meta_desc.get("content"):
+            desc = meta_desc["content"].strip()
+            print("Meta description:", desc[:300])
+
+        # Headings
+        h1s = [h.get_text(strip=True) for h in soup.find_all("h1")]
+        h2s = [h.get_text(strip=True) for h in soup.find_all("h2")]
+        print("H1s:", h1s[:5])
+        print("H2s:", h2s[:8])
+
+        # Nav / header anchors
+        navs = soup.find_all("nav")
+        if navs:
+            print(f"Found {len(navs)} <nav> block(s). Sample nav links:")
+            nav_links = []
+            for nav in navs:
+                for a in nav.find_all("a", href=True):
+                    nav_links.append((a.get_text(strip=True), self.abs_link(url, a["href"])))
+            for t, link in nav_links[:20]:
+                print(" -", t or "<no-text>", "->", link)
+        else:
+            print("No <nav> block found (or it's rendered by JS).")
+
+        # Sample anchors across page
+        anchors = []
+        for a in soup.find_all("a", href=True):
+            text = a.get_text(strip=True)
+            href = a["href"].strip()
+            anchors.append((text, self.abs_link(url, href)))
+        anchors = [a for a in anchors if a[1] is not None]
+        print(f"Total anchors on page: {len(anchors)}. Showing first {min(print_anchors,len(anchors))}:")
+        for t, link in anchors[:print_anchors]:
+            print(" *", (t[:60] or "<no-text>"), "->", link)
+
+        # Class name frequencies
+        classes = []
+        for tag in soup.find_all(True):
+            cls = tag.get("class")
+            if cls:
+                classes.extend(cls if isinstance(cls, list) else [cls])
+        class_counts = Counter(classes)
+        print("Top 15 classes used on page (class_name:count):")
+        for k,v in class_counts.most_common(15):
+            print("  ", k, ":", v)
+
+        # Links that look like files
+        file_like = []
+        for text, link in anchors:
+            if re.search(r"\.pdf$|\.xls$|\.xlsx$|\.csv$", link, re.IGNORECASE):
+                file_like.append((text, link))
+        print("File-like links found on page:", len(file_like))
+        for t, l in file_like[:20]:
+            print("  FILE:", (t[:80] or "<no-text>"), "->", l)
+
+        return {"title": title, "anchors": anchors, "file_links": file_like, "class_counts": class_counts}
+
+    def inspect_pages(self, urls):
+        results = {}
+        for u in urls:
+            print("\n" + "="*80)
+            print("Inspecting:", u)
+            out = self.explore_url(u, print_anchors=80)
+            results[u] = out
+            time.sleep(1.0)  # polite pause
+        return results
+
+    def collect_file_links(self, url, allowed_exts=(".pdf", ".xls", ".xlsx", ".csv")):
+        _, soup = self.fetch(url)
+        if not soup:
+            return pd.DataFrame()   # instead of returning []
+
+        found = []
+        for a in soup.find_all("a", href=True):
+            href = a["href"].strip()
+            ab = self.abs_link(url, href)
+            if not ab:
+                continue
+            # only same domain (safety)
+            if urlparse(ab).netloc.endswith("centralbank.go.ke") or urlparse(ab).netloc == "":
+                if any(ab.lower().endswith(ext) for ext in allowed_exts):
+                    found.append({"page":url, "text": a.get_text(strip=True), "file_url":ab})
+
+        # dedupe
+        seen = set()
+        dedup = []
+        for row in found:
+            if row["file_url"] not in seen:
+                dedup.append(row)
+                seen.add(row["file_url"])
+
+        df = pd.DataFrame(dedup)
+        print(f"Found {len(df)} file links on {url}")
+        return df
+
+
+    def crawl_links_for_files(self, start_url, allowed_exts=(".pdf", ".xls", ".xlsx", ".csv"), max_pages=50):
+        _, soup = self.fetch(start_url)
+        if not soup:
+            return []
+        pages = []
+        for a in soup.find_all("a", href=True):
+            href = a["href"].strip()
+            ab = self.abs_link(start_url, href)
+            if not ab:
+                continue
+            # only same domain
+            if urlparse(ab).netloc.endswith("centralbank.go.ke"):
+                pages.append(ab)
+        pages = list(dict.fromkeys(pages))[:max_pages]
+        print(f"Will inspect {len(pages)} linked pages from {start_url}")
+        results = []
+        for p in tqdm(pages):
+            df = self.collect_file_links(p, allowed_exts=allowed_exts)
+            if not df.empty:
+                results.append(df)
+            time.sleep(0.8)
+        if results:
+            return pd.concat(results, ignore_index=True)
+        return pd.DataFrame()
+
+def download_files(file_links, root_dir, save_dir,
+                   allowed_exts=(".pdf", ".xls", ".xlsx", ".csv"),
+                   overwrite=False):
+    """
+    Download multiple files from a list of (title, url) pairs.
+
+    Args:
+        file_links: list of (title, url) tuples, or list of dicts {"text":..., "file_url":...}
+        root_dir: base folder to save under
+        save_dir: subdirectory under root_dir
+        allowed_exts: file extensions to allow
+        overwrite: if True, re-download even if file exists
+
+    Returns:
+        metadata: list of dicts (title, url, local_path, size, status)
+    """
+    save_dir_path = os.path.join(root_dir, save_dir)
+    os.makedirs(save_dir_path, exist_ok=True)
+    metadata = []
+
+    # Normalize file_links into [(title, url), ...]
+    norm_links = []
+    for item in file_links:
+        if isinstance(item, tuple):
+            title, url = item
+        elif isinstance(item, dict):
+            title, url = item.get("text", "file"), item.get("file_url")
+        else:
+            continue
+        norm_links.append((title.strip(), url.strip()))
+
+    for title, url in norm_links:
+        # filter by extension
+        if not any(url.lower().endswith(ext) for ext in allowed_exts):
+            continue
+
+        # guess extension from URL
+        ext = os.path.splitext(urlparse(url).path)[1] or ".bin"
+        # clean filename
+        safe_title = re.sub(r"[^A-Za-z0-9._-]+", "_", title)[:100]
+        fname = f"{safe_title}{ext}"
+        path = os.path.join(save_dir_path, fname)
+
+        if os.path.exists(path) and not overwrite:
+            print(f"[skip] {fname} already exists.")
+            status = "skipped"
+        else:
+            try:
+                print(f"[download] {title} -> {fname}")
+                r = requests.get(url, stream=True, timeout=60)
+                r.raise_for_status()
+                with open(path, "wb") as f:
+                    for chunk in r.iter_content(8192):
+                        if chunk:
+                            f.write(chunk)
+                status = "ok"
+            except Exception as e:
+                print(f"[error] Failed: {url} ({e})")
+                status = "error"
+
+        size = os.path.getsize(path) if os.path.exists(path) else 0
+        metadata.append({
+            "title": title,
+            "url": url,
+            "local_path": path,
+            "size": size,
+            "status": status
+        })
+
+    return metadata
+
+import os, subprocess, importlib, sys
+
+def load_repo(repo):
+    local = repo.split("/")[-1]
+    if not os.path.exists(local):
+        subprocess.run(["git", "clone", f"https://github.com/{repo}.git"], check=True)
+    else:
+        subprocess.run(["git", "-C", local, "pull"], check=True)
+    if local not in sys.path:
+        sys.path.insert(0, local)
+    mod = importlib.import_module(local)
+    importlib.reload(mod)
+    return mod
+    
+def fetch_kenya_gnews(api_key):
+    # Free tier: 100 requests/day
+    url = f"https://gnews.io/api/v4/top-headlines?category=business&country=ke&token={api_key}"
+
+    response = requests.get(url)
+    data = response.json()
+
+    articles = []
+    for article in data.get('articles', []):
+        articles.append({
+            'title': article.get('title'),
+            'content': article.get('description'),
+            'url': article.get('url'),
+            'date': article.get('publishedAt'),
+            'source': article.get('source', {}).get('name')
+        })
+
+    df = pd.DataFrame(articles)
+    return df
+   
+def is_valid_url(url):
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except:
+        return False
+
+def fetch_kenya_thenewsapi(api_key):
+
+    url = f"https://api.thenewsapi.com/v1/news/all?api_token={api_key}&search=kenya+economy&language=en"
+
+    response = requests.get(url)
+    data = response.json()
+
+    articles = []
+    for article in data.get('data', []):
+        articles.append({
+            'title': article.get('title'),
+            'content': article.get('description'),
+            'url': article.get('url'),
+            'date': article.get('published_at'),
+            'source': article.get('source')
+        })
+
+    df = pd.DataFrame(articles)
+    return df
+
+def scrape_google_news_kenya():
+    google_news = GNews(
+        language='en',
+        country='KE',
+        period='7d',
+        max_results=50
+    )
+
+    # Search for Kenya business news
+    articles = google_news.get_news('Kenya economy OR inflation OR central bank')
+
+    df = pd.DataFrame(articles)
+    return df
+
+# Install: pip install gnews
+
+def scrape_african_business_rss():
+    feeds = [
+        'https://african.business/feed/',  # African Business Magazine
+        'https://www.cnbcafrica.com/feed/',  # CNBC Africa
+        'https://allafrica.com/tools/headlines/rdf/economy/headlines.rdf',  # AllAfrica Economy
+    ]
+
+    articles = []
+
+    for feed_url in feeds:
+        feed = feedparser.parse(feed_url)
+
+        for entry in feed.entries[:20]:
+            articles.append({
+                'title': entry.get('title', ''),
+                'url': entry.get('link', ''),
+                'date': entry.get('published', ''),
+                'summary': entry.get('summary', ''),
+                'source': feed.feed.get('title', '')
+            })
+
+    df = pd.DataFrame(articles)
+    return df
+
+def scrape_article(url: str, metadata: dict) -> dict:
+    """Scrape single article"""
+    try:
+        article = Article(url)
+        article.download()
+        article.parse()
+
+        if len(article.text) > 200:
+            return {
+                'title': article.title,
+                'full_content': article.text,
+                'summary': metadata.get('summary', ''),
+                'url': url,
+                'date': metadata.get('date'),
+                'source': metadata.get('source'),
+                'authors': ', '.join(article.authors) if article.authors else '',
+                'image': article.top_image,
+                'word_count': len(article.text.split()),
+                'status': 'success'
+            }
+        return None
+    except Exception as e:
+        return {'url': url, 'status': 'failed', 'error': str(e)}
+
+def fetch_newsdata_multi(api_key: str) -> List[Dict]:
+    """Multiple NewsData.io requests with pagination"""
+    all_articles = []
+
+    # Different queries to maximize coverage
+    queries = [
+        'kenya economy',
+        'kenya inflation',
+        'kenya central bank',
+        'kenya business',
+        'kenya finance'
+    ]
+
+    for query in queries:
+        try:
+            page = None
+            for _ in range(3):  # Up to 3 pages per query
+                params = {
+                    'apikey': api_key,
+                    'q': query,
+                    'country': 'ke',
+                    'language': 'en'
+                }
+                if page:
+                    params['page'] = page
+
+                response = requests.get('https://newsdata.io/api/1/latest', params=params, timeout=10)
+                data = response.json()
+
+                if data.get('status') != 'success':
+                    break
+
+                for item in data.get('results', []):
+                    if is_valid_url(item.get('link')):
+                        all_articles.append({
+                            'url': item.get('link'),
+                            'summary': item.get('description', ''),
+                            'date': item.get('pubDate'),
+                            'source': item.get('source_id')
+                        })
+
+                page = data.get('nextPage')
+                if not page:
+                    break
+
+                time.sleep(1)
+        except Exception as e:
+            print(f"NewsData query '{query}': {e}")
+            continue
+
+    return all_articles
+
+def fetch_gnews_multi(api_key: str) -> List[Dict]:
+    """Multiple GNews requests"""
+    all_articles = []
+
+    # Different search terms
+    searches = [
+        'kenya economy',
+        'kenya inflation',
+        'kenya business',
+        'nairobi stock exchange'
+    ]
+
+    for search in searches:
+        try:
+            params = {
+                'apikey': api_key,
+                'q': search,
+                'country': 'ke',
+                'lang': 'en',
+                'max': 10  # Free tier max
+            }
+
+            response = requests.get('https://gnews.io/api/v4/search', params=params, timeout=10)
+            data = response.json()
+
+            for item in data.get('articles', []):
+                if is_valid_url(item.get('url')):
+                    all_articles.append({
+                        'url': item.get('url'),
+                        'summary': item.get('description', ''),
+                        'date': item.get('publishedAt'),
+                        'source': item.get('source', {}).get('name')
+                    })
+
+            time.sleep(1)
+        except Exception as e:
+            print(f"GNews search '{search}': {e}")
+            continue
+
+    return all_articles
+
+def fetch_thenewsapi_multi(api_key: str) -> List[Dict]:
+    """Multiple TheNewsAPI requests (only 3 articles per request!)"""
+    all_articles = []
+
+    # Multiple searches to compensate for 3-article limit
+    searches = [
+        'kenya economy',
+        'kenya business',
+        'kenya inflation',
+        'kenya central bank',
+        'kenya finance',
+        'nairobi economy',
+        'kenya investment',
+        'kenya banking'
+    ]
+
+    for search in searches:
+        try:
+            params = {
+                'api_token': api_key,
+                'search': search,
+                'language': 'en',
+                'limit': 3  # Free tier limit
+            }
+
+            response = requests.get('https://api.thenewsapi.com/v1/news/all', params=params, timeout=10)
+            data = response.json()
+
+            for item in data.get('data', []):
+                if is_valid_url(item.get('url')):
+                    all_articles.append({
+                        'url': item.get('url'),
+                        'summary': item.get('description', ''),
+                        'date': item.get('published_at'),
+                        'source': item.get('source')
+                    })
+
+            time.sleep(1)
+        except Exception as e:
+            print(f"TheNewsAPI search '{search}': {e}")
+            continue
+
+    return all_articles
+
+def scrape_kenya_news_maximum(
+    newsdata_key: str = None,
+    gnews_key: str = None,
+    thenewsapi_key: str = None,
+    max_workers: int = 8
+) -> pd.DataFrame:
+    """Get MAXIMUM articles from all sources"""
+
+    print("🔍 Fetching maximum articles from all APIs...\n")
+
+    all_articles = []
+
+    # Fetch from all sources
+    if newsdata_key:
+        print("📰 NewsData.io: ", end="", flush=True)
+        articles = fetch_newsdata_multi(newsdata_key)
+        all_articles.extend(articles)
+        print(f"{len(articles)} URLs")
+
+    if gnews_key:
+        print("📰 GNews.io: ", end="", flush=True)
+        articles = fetch_gnews_multi(gnews_key)
+        all_articles.extend(articles)
+        print(f"{len(articles)} URLs")
+
+    if thenewsapi_key:
+        print("📰 TheNewsAPI: ", end="", flush=True)
+        articles = fetch_thenewsapi_multi(thenewsapi_key)
+        all_articles.extend(articles)
+        print(f"{len(articles)} URLs (limited to 3/request on free)")
+
+    if not all_articles:
+        print("\n No articles found")
+        return pd.DataFrame()
+
+    # Deduplicate by URL
+    seen = set()
+    unique = []
+    for a in all_articles:
+        if a['url'] not in seen:
+            seen.add(a['url'])
+            unique.append(a)
+
+    print(f"\n Total unique URLs: {len(unique)}\n")
+
+    # Parallel scraping
+    results = []
+    failed = 0
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(scrape_article, a['url'], a): a for a in unique}
+
+        with tqdm(total=len(futures), desc="📄 Scraping", unit="article") as pbar:
+            for future in as_completed(futures):
+                result = future.result()
+
+                if result and result.get('status') == 'success':
+                    results.append(result)
+                elif result:
+                    failed += 1
+                    if failed <= 3:  # Only show first 3 errors
+                        print(f"\n {result['url'][:50]}... | {result['error']}")
+
+                pbar.update(1)
+                time.sleep(0.2)
+
+    # Save
+    if results:
+        df = pd.DataFrame(results)
+        df = df.drop('status', axis=1, errors='ignore')
+
+        print(f"\n {len(results)} articles scraped | {failed} failed | {len(results)/(len(results)+failed)*100:.1f}% success")
+        print(f" Avg: {df['word_count'].mean():.0f} words | {df['source'].nunique()} sources")
+        return df
+
+    return pd.DataFrame()

src/load/clean_db.py

@@ -0,0 +1,244 @@
+# clean_database.py
+import pandas as pd
+from sqlalchemy import create_engine, text
+import logging
+
+# Set up a logger that can be configured by the importer
+logger = logging.getLogger("DBCleaner")
+
+def drop_blacklisted_tables(engine):
+    """Drops tables matching the blacklist patterns."""
+    drop_patterns = [
+        "bop_annual",
+        "commercial_banks_average_lending_rates",
+        "depository_corporation", 
+        "exchange_rates_end_period",
+        "exchange_rates_period_average",
+        "forex_bureau_rates_sheet", 
+        "lr_return_template",
+        "nsfr_return_template"
+    ]
+    
+    with engine.connect() as conn:
+        all_tables = [t[0] for t in conn.execute(text("SELECT name FROM sqlite_master WHERE type='table'")).fetchall()]
+        tables_to_drop = []
+        
+        for t in all_tables:
+            if any(p in t for p in drop_patterns):
+                tables_to_drop.append(t)
+        
+        if not tables_to_drop:
+            logger.info("No tables found matching blacklist patterns.")
+            return
+
+        logger.info(f"🗑️  Dropping {len(tables_to_drop)} tables...")
+        for t in tables_to_drop:
+            conn.execute(text(f'DROP TABLE "{t}"'))
+            logger.info(f"   - Dropped: {t}")
+        conn.commit()
+
+def clean_table(engine, table_name, drop_top_rows=0, rename_map=None, rename_by_index=None, static_date=None):
+    """
+    Generic cleaner for specific table fixes.
+    """
+    try:
+        # Check if table exists first
+        with engine.connect() as conn:
+            exists = conn.execute(text(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'")).scalar()
+        if not exists:
+            logger.warning(f"   Table '{table_name}' not found. Skipping.")
+            return
+
+        df = pd.read_sql(f'SELECT * FROM "{table_name}"', engine)
+        if df.empty: return
+
+        # Drop columns that are completely empty
+        df = df.dropna(axis=1, how='all')
+        
+        # Drop top rows if requested
+        if drop_top_rows > 0:
+            df = df.iloc[drop_top_rows:].reset_index(drop=True)
+            
+        # Rename by Index (useful for 'col_1', 'col_2')
+        if rename_by_index:
+            curr_cols = list(df.columns)
+            new_cols = curr_cols.copy()
+            for idx, new_name in rename_by_index.items():
+                if idx < len(curr_cols):
+                    new_cols[idx] = new_name
+            df.columns = new_cols
+            
+        # Rename by Map
+        if rename_map:
+            df.rename(columns=rename_map, inplace=True)
+            
+        # Inject Static Date if missing
+        if static_date:
+            if 'date' not in df.columns:
+                df.insert(0, 'date', static_date)
+            else:
+                df['date'] = static_date
+
+        # Save back to DB (Replace mode)
+        df.to_sql(table_name, engine, if_exists='replace', index=False)
+        logger.info(f"   Fixed '{table_name}': {len(df)} rows")
+        
+    except Exception as e:
+        logger.error(f"    Error cleaning '{table_name}': {e}")
+
+def run_specific_fixes(engine):
+    """Orchestrates the specific cleaning rules."""
+    logger.info("🔧 Running specific table fixes...")
+    
+    # 1. Historical Rates
+    clean_table(engine, "download_all_historical_rates", 
+                rename_by_index={2: "mean_rate", 3: "buy_rate", 4: "sell_rate"})
+
+    # 2. Foreign Trade Summary
+    clean_table(engine, "foreign_trade_summary", drop_top_rows=1)
+
+    # 3. Forex Bureau Rates
+    clean_table(engine, "forex_bureau_rates", 
+                rename_map={"bureau_name": "currency"})
+
+    # 4. Indicative Rates (Indicative Sheet)
+    clean_table(engine, "indicative_rates_sheet_indicative", 
+                static_date="2017-11-16",
+                rename_by_index={0: "currency", 1: "mean_rate", 2: "buy_rate", 3: "sell_rate"})
+
+    # 5. Indicative Rates (Press Sheet)
+    clean_table(engine, "indicative_rates_sheet_press", 
+                static_date="2017-11-16",
+                rename_by_index={
+                    0: "bank_name",
+                    1: "usd_buy", 2: "usd_sell", 3: "usd_margin",
+                    4: "gbp_buy", 5: "gbp_sell", 6: "gbp_margin"
+                })
+
+    # 6. Selected Domestic Exports
+    clean_table(engine, "value_of_selected_domestic_exports", drop_top_rows=2)
+
+    # 7. Imports by Commodity
+    clean_table(engine, "value_of_direct_imports_by_commodities", drop_top_rows=1)
+
+def clean_database_pipeline(db_name):
+    """Main entry point for external calls."""
+    connection_str = f"sqlite:///{db_name}"
+    engine = create_engine(connection_str)
+    
+    logger.info(f" Starting cleanup on {db_name}...")
+    drop_blacklisted_tables(engine)
+    run_specific_fixes(engine)
+    logger.info(" Cleanup Complete.")
+
+def drop_tables(engine):
+    """Drops the specific list of tables requested."""
+    tables_to_drop = [
+        'forex_bureau_rates',
+        'forex_bureaus_rates_sheet_chief_dealers',
+        'forex_bureaus_rates_sheet_director',
+        'forex_bureaus_rates_sheet_directors',
+        'forex_bureaus_rates_sheet_fbx',
+        'forex_bureaus_rates_sheet_fbx1',
+        'forex_bureaus_rates_sheet_fbx2',
+        'forex_bureaus_rates_sheet_fxb1',
+        'forex_bureaus_rates_sheet_fxb2',
+        'forex_bureaus_rates_sheet_fxb22',
+        'forex_bureaus_rates_sheet_market_intelligence',
+        'forex_bureaus_rates_sheet_sheet1',
+        'forex_bureaus_rates_sheet_sheet2',
+        'forex_bureaus_rates_sheet_sheet3',
+        'forex_bureaus_rates_sheet_sheet4',
+        'issues_of_treasury_bills',
+        'issues_of_treasury_bonds'
+    ]
+
+    print("🗑️  Dropping Tables...")
+    with engine.connect() as conn:
+        for t in tables_to_drop:
+            try:
+                conn.execute(text(f'DROP TABLE IF EXISTS "{t}"'))
+                print(f"   - Dropped: {t}")
+            except Exception as e:
+                print(f"   Could not drop {t}: {e}")
+        conn.commit()
+
+def fix_foreign_trade(engine):
+    """Renames first column to 'year'."""
+    table_name = "foreign_trade_summary"
+    try:
+        df = pd.read_sql(f'SELECT * FROM "{table_name}"', engine)
+        if 'kenyan_shillings_million_year' in df.columns:
+            df.rename(columns={'kenyan_shillings_million_year': 'year'}, inplace=True)
+            df.to_sql(table_name, engine, if_exists='replace', index=False)
+            print(f" Fixed '{table_name}': Renamed 'year' column.")
+        else:
+            print(f"  '{table_name}': Target column not found.")
+    except Exception as e:
+        print(f" Error fixing {table_name}: {e}")
+
+def fix_indicative_rates_shift(engine):
+    """
+    Applies the 'Shift Right + Fixed Date' logic.
+    Inserts 2017-11-16 at position 0, shifting existing data to the right.
+    """
+    targets = [
+        "indicative_rates_sheet_indicative",
+        "indicative_rates_sheet_press"
+    ]
+    
+    fixed_date = "2017-11-16"
+
+    for table in targets:
+        try:
+            df = pd.read_sql(f'SELECT * FROM "{table}"', engine)
+            if df.empty: continue
+
+            # Logic: Insert new date column at index 0
+            # This effectively "shifts" the old col 0 to col 1
+            df.insert(0, 'fixed_date', fixed_date)
+            
+            # Rename columns to reflect the shift clearly
+            # We assume the user wants standard names for the shifted data
+            # Adjust names based on the table type
+            new_columns = list(df.columns)
+            new_columns[0] = "date" # The new fixed column
+            
+            # Assigning generic or specific headers for the shifted data
+            if "press" in table:
+                # Based on previous prompt instructions for Press sheet:
+                # Bank, USD_Buy, USD_Sell, USD_Margin, GBP_Buy...
+                expected_headers = ["date", "bank_name", "usd_buy", "usd_sell", "usd_margin", "gbp_buy", "gbp_sell", "gbp_margin", "euro_buy", "euro_sell", "euro_margin"]
+            else:
+                # Indicative sheet: Currency, Mean, Buy, Sell
+                expected_headers = ["date", "currency", "mean_rate", "buy_rate", "sell_rate"]
+
+            # Map headers safely (truncate if df has fewer cols, pad if more)
+            final_cols = expected_headers + [f"col_{i}" for i in range(len(df.columns) - len(expected_headers))]
+            df.columns = final_cols[:len(df.columns)]
+            
+            # Clean up: Drop any old 'date' column if it was pushed to the right and is duplicate/garbage
+            # (Optional, but safer to keep strictly what we shifted)
+            
+            df.to_sql(table, engine, if_exists='replace', index=False)
+            print(f" Fixed '{table}': Applied Date Shift & Header Rename.")
+            
+        except Exception as e:
+            print(f" Error fixing {table}: {e}")
+
+def fix_cbk_indicative_swap(engine):
+    """Swaps 'date' and 'currency' column names."""
+    table_name = "cbk_indicative_rates"
+    try:
+        df = pd.read_sql(f'SELECT * FROM "{table_name}"', engine)
+        
+        rename_map = {}
+        if 'date' in df.columns: rename_map['date'] = 'currency'
+        if 'currency' in df.columns: rename_map['currency'] = 'date'
+        
+        if rename_map:
+            df.rename(columns=rename_map, inplace=True)
+            df.to_sql(table_name, engine, if_exists='replace', index=False)
+            print(f" Fixed '{table_name}': Swapped 'date' <-> 'currency'.")
+    except Exception as e:
+        print(f" Error fixing {table_name}: {e}")

src/load/explore_news_schema.py

@@ -0,0 +1,74 @@
+import pandas as pd
+import glob
+import os
+import logging
+import sys
+
+# Configure logging to show up in the notebook
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(message)s', 
+    stream=sys.stdout, 
+    force=True
+)
+logger = logging.getLogger("SchemaExplorer")
+
+def analyze_schemas(news_dir: str):
+    """
+    Scans all CSV files in the given directory and groups them by their column structure.
+    """
+    if not os.path.exists(news_dir):
+        logger.error(f" Directory not found: {news_dir}")
+        return
+
+    csv_files = glob.glob(os.path.join(news_dir, "*.csv"))
+    logger.info(f"🔍 Scanning {len(csv_files)} files in '{news_dir}'...\n")
+    
+    if not csv_files:
+        logger.warning(" No CSV files found.")
+        return
+
+    # Dictionary to store unique schemas: { (col1, col2): [file1, file2] }
+    schemas = {}
+    
+    for f in csv_files:
+        try:
+            # Read only the header (fast)
+            df = pd.read_csv(f, nrows=0)
+            
+            # Sort columns to ensure order doesn't matter for grouping
+            cols = tuple(sorted(df.columns.tolist()))
+            
+            if cols not in schemas:
+                schemas[cols] = []
+            schemas[cols].append(os.path.basename(f))
+            
+        except Exception as e:
+            logger.error(f" Error reading {os.path.basename(f)}: {e}")
+
+    # Report Findings
+    logger.info("--- Schema Report ---")
+    for i, (cols, files) in enumerate(schemas.items()):
+        logger.info(f"\nTYPE {i+1}: Found in {len(files)} files")
+        logger.info(f"Columns: {list(cols)}")
+        if len(files) < 5:
+            logger.info(f"Examples: {files}")
+        else:
+            logger.info(f"Examples: {files[:3]} ... (+{len(files)-3} others)")
+
+    # Date Format Check (Random Sample from the first valid file)
+    logger.info("\n--- Date Format Sample ---")
+    try:
+        sample_file = csv_files[0]
+        sample = pd.read_csv(sample_file, nrows=5)
+        
+        # Look for a column containing 'date' or 'time'
+        date_col = next((c for c in sample.columns if 'date' in c.lower() or 'time' in c.lower() or 'published' in c.lower()), None)
+        
+        if date_col:
+            logger.info(f"Sample from column '{date_col}' in {os.path.basename(sample_file)}:")
+            logger.info(sample[date_col].head().tolist())
+        else:
+            logger.warning("No obvious 'date' column found in sample.")
+    except Exception as e:
+        logger.error(f"Could not read sample for date check: {e}")

src/load/ingest_md.py

@@ -0,0 +1,117 @@
+import os
+import logging
+import sys
+import time
+import requests
+import subprocess
+from pathlib import Path
+from tqdm import tqdm
+from langchain_community.document_loaders import DirectoryLoader, TextLoader # <--- SWITCHED
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import OllamaEmbeddings
+
+# --- LOGGING ---
+logging.basicConfig(level=logging.INFO, stream=sys.stdout, format='%(message)s', force=True)
+logger = logging.getLogger("ReportIngest")
+
+def _ensure_ollama_running(port="25000"):
+    host = f"http://127.0.0.1:{port}"
+    try:
+        if requests.get(host).status_code == 200:
+            return True
+    except: pass
+    
+    print(" Starting Ollama Server...")
+    scratch = os.environ.get("SCRATCH", "/tmp")
+    base = Path(scratch)
+    bin_path = base / "ollama_core/bin/ollama"
+    
+    env = os.environ.copy()
+    env["OLLAMA_HOST"] = f"127.0.0.1:{port}"
+    env["OLLAMA_MODELS"] = str(base / "ollama_core/models")
+    
+    subprocess.Popen([str(bin_path), "serve"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env)
+    time.sleep(5)
+    return True
+
+def ingest_markdown_reports(
+    markdown_dir="mshauri-fedha/data/knbs/marker-output",
+    vector_db_path="mshauri_fedha_chroma_db",
+    model="nomic-embed-text",
+    ollama_port="25000"
+):
+    _ensure_ollama_running(ollama_port)
+
+    if not os.path.exists(markdown_dir):
+        logger.error(f" Directory not found: {markdown_dir}")
+        return
+
+    print(f"📄 Scanning for Markdown Reports in {markdown_dir}...")
+    
+    # --- 1. LOAD FILES (Improved) ---
+    # We use TextLoader which is faster and doesn't trigger 'unstructured' warnings
+    loader = DirectoryLoader(
+        markdown_dir,
+        glob="**/*.md",
+        loader_cls=TextLoader,
+        loader_kwargs={'autodetect_encoding': True}, # Safe for varying file encodings
+        show_progress=True,
+        use_multithreading=True
+    )
+    
+    # Catch errors during loading (e.g., empty files)
+    try:
+        raw_docs = loader.load()
+    except Exception as e:
+        print(f" Warning during loading: {e}")
+        # Fallback: simple load if directory loader fails
+        raw_docs = []
+
+    if not raw_docs:
+        print(" No valid markdown files found.")
+        return
+
+    print(f"   Loaded {len(raw_docs)} report files.")
+
+    # --- 2. CHUNKING ---
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=2000, 
+        chunk_overlap=200,
+        separators=["\n## ", "\n### ", "\n", " ", ""]
+    )
+    docs = text_splitter.split_documents(raw_docs)
+    
+    # --- 3. METADATA ---
+    for d in docs:
+        d.metadata["type"] = "report"
+        if "source" not in d.metadata:
+            d.metadata["source"] = os.path.basename(d.metadata.get("source", "Official Report"))
+
+    print(f"   ✂️  Split into {len(docs)} chunks.")
+
+    # --- 4. EMBEDDING ---
+    print(" Appending to Vector Store...")
+    embeddings = OllamaEmbeddings(
+        model=model,
+        base_url=f"http://127.0.0.1:{ollama_port}"
+    )
+    
+    vectorstore = Chroma(
+        persist_directory=vector_db_path,
+        embedding_function=embeddings
+    )
+    
+    # Batch Add
+    batch_size = 100
+    
+    with tqdm(total=len(docs), desc="Ingesting Reports", unit="chunk") as pbar:
+        for i in range(0, len(docs), batch_size):
+            batch = docs[i:i+batch_size]
+            vectorstore.add_documents(batch)
+            pbar.update(len(batch))
+
+    print("\n Reports Added. Hybrid Knowledge Base is ready.")
+
+if __name__ == "__main__":
+    ingest_markdown_reports()

src/load/ingest_news.py

@@ -0,0 +1,162 @@
+import pandas as pd
+import os
+import re
+import ast
+import logging
+import glob
+import time
+import requests
+import subprocess
+import sys
+from pathlib import Path
+from tqdm import tqdm # <--- New Import
+from dateutil import parser
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import OllamaEmbeddings
+
+# --- CONFIG ---
+MIN_CONTENT_LENGTH = 100 
+
+# --- LOGGING ---
+logging.basicConfig(level=logging.INFO, stream=sys.stdout, format='%(message)s', force=True)
+logger = logging.getLogger("NewsIngest")
+
+def _ensure_ollama_running(port="25000"):
+    host = f"http://127.0.0.1:{port}"
+    try:
+        if requests.get(host).status_code == 200:
+            return True
+    except: pass
+    
+    print(" Starting Ollama Server...")
+    scratch = os.environ.get("SCRATCH", "/tmp")
+    base = Path(scratch)
+    bin_path = base / "ollama_core/bin/ollama"
+    
+    env = os.environ.copy()
+    env["OLLAMA_HOST"] = f"127.0.0.1:{port}"
+    env["OLLAMA_MODELS"] = str(base / "ollama_core/models")
+    
+    subprocess.Popen([str(bin_path), "serve"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env)
+    time.sleep(5)
+    return True
+
+# --- CLEANING HELPERS ---
+def clean_text(text):
+    if not isinstance(text, str): return ""
+    text = re.sub(r'(?:https?|ftp)://\S+|www\.\S+', '', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    text = re.sub(r'^[\W_]+', '', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def parse_standard_date(date_str):
+    try:
+        if pd.isna(date_str): return "Unknown Date"
+        dt = parser.parse(str(date_str))
+        return dt.strftime("%Y-%m-%d")
+    except: return "Unknown Date"
+
+def extract_publisher_from_dict(pub_str):
+    try:
+        if isinstance(pub_str, str) and "{" in pub_str:
+            data = ast.literal_eval(pub_str)
+            return data.get('title', 'Google News')
+        return str(pub_str)
+    except: return "Google News"
+
+def normalize_news_df(df, filename):
+    cols = df.columns.tolist()
+    normalized = []
+
+    def create_entry(row, title_col, content_col, date_col, source_val):
+        title = clean_text(row.get(title_col, ''))
+        content = clean_text(row.get(content_col, ''))
+        if len(content) < MIN_CONTENT_LENGTH: return None
+        return {
+            'title': title,
+            'content': content,
+            'date': parse_standard_date(row.get(date_col, '')),
+            'source': source_val,
+            'url': row.get('url', ''),
+            'file_origin': filename
+        }
+
+    if 'publisher' in cols and 'description' in cols:
+        for _, row in df.iterrows():
+            source = extract_publisher_from_dict(row.get('publisher', ''))
+            entry = create_entry(row, 'title', 'description', 'published date', source)
+            if entry: normalized.append(entry)
+    elif 'full_content' in cols:
+        for _, row in df.iterrows():
+            c_col = 'full_content' if isinstance(row.get('full_content'), str) and len(str(row.get('full_content'))) > 50 else 'summary'
+            entry = create_entry(row, 'title', c_col, 'date', str(row.get('source', 'Unknown')))
+            if entry: normalized.append(entry)
+    elif 'content' in cols and 'source' in cols:
+        for _, row in df.iterrows():
+            entry = create_entry(row, 'title', 'content', 'date', str(row.get('source', 'Unknown')))
+            if entry: normalized.append(entry)
+    return pd.DataFrame(normalized)
+
+def ingest_news_data(news_dir, vector_db_path="mshauri_fedha_chroma_db", model="nomic-embed-text"):
+    _ensure_ollama_running()
+
+    csv_files = glob.glob(os.path.join(news_dir, "*.csv"))
+    if not csv_files:
+        print("No files found.")
+        return
+
+    print(f" Found {len(csv_files)} news files. Processing...")
+    
+    all_articles = []
+    
+    # Progress bar for loading files
+    for f in tqdm(csv_files, desc="Reading CSVs", unit="file"):
+        try:
+            df = pd.read_csv(f)
+            clean_df = normalize_news_df(df, os.path.basename(f))
+            if not clean_df.empty:
+                all_articles.extend(clean_df.to_dict('records'))
+        except Exception as e:
+            pass
+
+    # Deduplication
+    unique_docs = {}
+    for art in all_articles:
+        key = f"{art['title']}_{art['date']}"
+        if art['title'] in art['content']:
+            page_content = f"Date: {art['date']}\nSource: {art['source']}\n\n{art['content']}"
+        else:
+            page_content = f"Title: {art['title']}\nDate: {art['date']}\nSource: {art['source']}\n\n{art['content']}"
+        
+        if key in unique_docs:
+            if len(page_content) > len(unique_docs[key].page_content):
+                unique_docs[key] = Document(page_content=page_content, metadata={"source": art['source'], "date": art['date'], "type": "news"})
+        else:
+            unique_docs[key] = Document(page_content=page_content, metadata={"source": art['source'], "date": art['date'], "type": "news"})
+            
+    raw_docs = list(unique_docs.values())
+    print(f"   Condensed into {len(raw_docs)} unique articles.")
+
+    # Chunking
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=200, separators=["\n\n", "\n", ". ", " "])
+    final_docs = text_splitter.split_documents(raw_docs)
+    
+    if final_docs:
+        print(f" Embedding {len(final_docs)} chunks into Vector DB...")
+        embeddings = OllamaEmbeddings(model=model, base_url="http://127.0.0.1:25000")
+        vectorstore = Chroma(persist_directory=vector_db_path, embedding_function=embeddings)
+        
+        batch_size = 100
+        # Progress bar for embedding
+        with tqdm(total=len(final_docs), desc="Embedding News", unit="chunk") as pbar:
+            for i in range(0, len(final_docs), batch_size):
+                batch = final_docs[i:i+batch_size]
+                vectorstore.add_documents(batch)
+                pbar.update(len(batch))
+            
+        print("\n News Ingestion Complete.")
+    else:
+        print("No valid articles extracted.")

src/load/inspect_db.py

@@ -0,0 +1,92 @@
+import os
+import pandas as pd
+from sqlalchemy import create_engine, text
+
+# --- CONFIGURATION ---
+DB_NAME = "mshauri_fedha.db"
+DB_CONNECTION = f"sqlite:///{DB_NAME}"
+
+def list_all_tables(engine):
+    print(f"\n --- DATABASE SUMMARY: {DB_NAME} ---")
+    try:
+        with engine.connect() as conn:
+            # Query the master table for all table names
+            query = text("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;")
+            tables = conn.execute(query).fetchall()
+            
+            if not tables:
+                print(" Database is empty.")
+                return []
+
+            table_list = [t[0] for t in tables]
+            
+            print(f"{'ID':<4} | {'Rows':<8} | {'Table Name'}")
+            print("-" * 60)
+            
+            for i, t_name in enumerate(table_list):
+                # Count rows for verification
+                try:
+                    count = conn.execute(text(f'SELECT COUNT(*) FROM "{t_name}"')).scalar()
+                    print(f"{i:<4} | {count:<8} | {t_name}")
+                except:
+                    print(f"{i:<4} | {'ERROR':<8} | {t_name}")
+            
+            return table_list
+    except Exception as e:
+        print(f" Connection failed: {e}")
+        return []
+
+def inspect_table(engine, table_name):
+    print(f"\n🔎 Inspecting Table: '{table_name}'")
+    try:
+        # Read schema/columns
+        query = f'SELECT * FROM "{table_name}" LIMIT 5'
+        df = pd.read_sql(query, engine)
+        
+        if df.empty:
+            print(" Table is empty.")
+        else:
+            print(f"Columns: {list(df.columns)}")
+            print("\n--- First 5 Rows ---")
+            # to_string() makes it readable in terminal without truncation
+            print(df.to_string(index=False)) 
+            print("-" * 50)
+    except Exception as e:
+        print(f" Could not read table: {e}")
+
+def main():
+    if not os.path.exists(DB_NAME):
+        print(f" Error: Database file '{DB_NAME}' not found in current directory.")
+        print(f"Current Directory: {os.getcwd()}")
+        return
+
+    engine = create_engine(DB_CONNECTION)
+    tables = list_all_tables(engine)
+    
+    if not tables: return
+
+    while True:
+        try:
+            user_input = input("\nEnter Table ID (or Name) to inspect, or 'q' to quit: ").strip()
+            if user_input.lower() == 'q': break
+            
+            target_table = None
+            
+            # Handle numeric ID input
+            if user_input.isdigit():
+                idx = int(user_input)
+                if 0 <= idx < len(tables):
+                    target_table = tables[idx]
+            # Handle name input
+            elif user_input in tables:
+                target_table = user_input
+            
+            if target_table:
+                inspect_table(engine, target_table)
+            else:
+                print(" Invalid selection.")
+        except KeyboardInterrupt:
+            break
+
+if __name__ == "__main__":
+    main()

src/load/mshauri_demo.py

@@ -0,0 +1,198 @@
+import os
+import re
+import sys
+# These imports are stable and have worked in your previous logs
+from langchain_ollama import ChatOllama
+from langchain_community.utilities import SQLDatabase
+from langchain_community.agent_toolkits.sql.toolkit import SQLDatabaseToolkit
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import OllamaEmbeddings
+
+# --- CONFIGURATION ---
+DEFAULT_SQL_DB = "sqlite:///mshauri_fedha_v6.db"
+DEFAULT_VECTOR_DB = "mshauri_fedha_chroma_db"
+DEFAULT_EMBED_MODEL = "nomic-embed-text"
+DEFAULT_LLM_MODEL = "qwen3:32b"
+DEFAULT_OLLAMA_URL = "http://127.0.0.1:25000"
+
+# --- 1. REPLACEMENT CLASS FOR 'Tool' ---
+class SimpleTool:
+    """A simple wrapper to replace langchain.tools.Tool"""
+    def __init__(self, name, func, description):
+        self.name = name
+        self.func = func
+        self.description = description
+    
+    def run(self, input_data):
+        return self.func(input_data)
+
+# --- 2. REPLACEMENT CLASS FOR THE AGENT ---
+class SimpleReActAgent:
+    """A manual ReAct loop that doesn't rely on langchain.agents"""
+    def __init__(self, llm, tools, verbose=True):
+        self.llm = llm
+        self.tools = {t.name: t for t in tools}
+        self.verbose = verbose
+        # Create the tool description string for the prompt
+        self.tool_desc = "\n".join([f"{t.name}: {t.description}" for t in tools])
+        self.tool_names = ", ".join([t.name for t in tools])
+        
+        # Hardcoded ReAct Prompt
+        self.prompt_template = """Answer the following questions as best you can. You have access to the following tools:
+
+{tool_desc}
+
+Use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action: the action to take, should be one of [{tool_names}]
+Action Input: the input to the action
+Observation: the result of the action
+... (this Thought/Action/Action Input/Observation can repeat N times)
+Thought: I now know the final answer
+Final Answer: the final answer to the original input question
+
+Begin!
+
+Question: {input}
+Thought:{agent_scratchpad}"""
+
+    def invoke(self, inputs):
+        query = inputs["input"]
+        scratchpad = ""
+        
+        print(f" Starting Agent Loop for: '{query}'")
+        
+        for step in range(10): # Max 10 steps
+            # Fill the prompt
+            prompt = self.prompt_template.format(
+                tool_desc=self.tool_desc,
+                tool_names=self.tool_names,
+                input=query,
+                agent_scratchpad=scratchpad
+            )
+            
+            # Call LLM
+            # stop=["\nObservation:"] prevents the LLM from hallucinating the tool output
+            response = self.llm.invoke(prompt, stop=["\nObservation:"])
+            response_text = response.content
+            
+            if self.verbose:
+                print(f"\n🧠 Step {step+1}: {response_text.strip()}")
+
+            scratchpad += response_text
+
+            # Check for completion
+            if "Final Answer:" in response_text:
+                return {"output": response_text.split("Final Answer:")[-1].strip()}
+
+            # Parse Action
+            action_match = re.search(r"Action:\s*(.*?)\n", response_text)
+            input_match = re.search(r"Action Input:\s*(.*)", response_text)
+            
+            if action_match and input_match:
+                action_name = action_match.group(1).strip()
+                action_input = input_match.group(1).strip()
+                
+                # Execute Tool
+                if action_name in self.tools:
+                    if self.verbose:
+                        print(f"🛠️  Calling '{action_name}' with: {action_input}")
+                    
+                    try:
+                        # Handle both SimpleTool (.run) and LangChain Tools (.invoke or .run)
+                        tool = self.tools[action_name]
+                        if hasattr(tool, 'invoke'):
+                            tool_result = tool.invoke(action_input)
+                        else:
+                            tool_result = tool.run(action_input)
+                            
+                    except Exception as e:
+                        tool_result = f"Error executing tool: {e}"
+                        
+                    observation = f"\nObservation: {tool_result}\n"
+                else:
+                    observation = f"\nObservation: Error: Tool '{action_name}' not found. Available: {self.tool_names}\n"
+                
+                scratchpad += observation
+            else:
+                # Fallback: if no action found but also no Final Answer
+                if "Action:" in response_text:
+                    scratchpad += "\nObservation: You provided an Action but no Action Input. Please provide the input.\n"
+                else:
+                    return {"output": response_text.strip()}
+                    
+        return {"output": "Agent timed out."}
+
+# --- 3. MAIN SETUP FUNCTION ---
+
+def create_mshauri_agent(
+    sql_db_path=DEFAULT_SQL_DB,
+    vector_db_path=DEFAULT_VECTOR_DB,
+    llm_model=DEFAULT_LLM_MODEL,
+    ollama_url=DEFAULT_OLLAMA_URL
+):
+    print(f"  Initializing Mshauri Fedha (Model: {llm_model})...")
+    
+    # 1. Initialize LLM
+    try:
+        llm = ChatOllama(model=llm_model, base_url=ollama_url, temperature=0.1)
+    except Exception as e:
+        print(f" Error connecting to Ollama: {e}")
+        return None
+
+    # 2. LEFT BRAIN (SQL)
+    if "sqlite" in sql_db_path:
+        real_path = sql_db_path.replace("sqlite:///", "")
+        if not os.path.exists(real_path):
+             print(f"  Warning: SQL Database not found at {real_path}")
+
+    db = SQLDatabase.from_uri(sql_db_path)
+    # The Toolkit returns standard LangChain tools, which our SimpleReActAgent can handle
+    sql_toolkit = SQLDatabaseToolkit(db=db, llm=llm)
+    sql_tools = sql_toolkit.get_tools()
+
+    # 3. RIGHT BRAIN (Vector)
+    # We define the Retriever function manually
+    def search_docs(query):
+        embeddings = OllamaEmbeddings(model=DEFAULT_EMBED_MODEL, base_url=ollama_url)
+        vectorstore = Chroma(persist_directory=vector_db_path, embedding_function=embeddings)
+        docs = vectorstore.similarity_search(query, k=4)
+        return "\n\n".join([d.page_content for d in docs])
+
+    # Use our SimpleTool wrapper instead of importing from langchain
+    retriever_tool = SimpleTool(
+        name="search_financial_reports_and_news",
+        func=search_docs,
+        description="Searches CBK/KNBS reports and business news. Use this for qualitative questions (why, how, trends) or when SQL data is missing."
+    )
+
+    # 4. CREATE AGENT
+    tools = sql_tools + [retriever_tool]
+    agent = SimpleReActAgent(llm, tools)
+    
+    print(" Mshauri Agent Ready (Zero-Dependency Mode).")
+    return agent
+
+def ask_mshauri(agent, query):
+    if not agent:
+        print(" Agent not initialized.")
+        return
+
+    print(f"\n User: {query}")
+    print("-" * 40)
+    
+    try:
+        response = agent.invoke({"input": query})
+        print("-" * 40)
+        print(f" Mshauri: {response['output']}")
+        return response['output']
+    except Exception as e:
+        print(f" Error during execution: {e}")
+        return None
+
+if __name__ == "__main__":
+    # Quick Test
+    agent = create_mshauri_agent()
+    ask_mshauri(agent, "What is the inflation rate?")

src/load/start_ollama.py

@@ -0,0 +1,84 @@
+import os
+import subprocess
+import time
+import requests
+from pathlib import Path
+
+def start_ollama_server():
+    """Checks if Ollama is running on port 25000, if not, starts it."""
+    OLLAMA_PORT = "25000"
+    OLLAMA_HOST = f"http://127.0.0.1:{OLLAMA_PORT}"
+    
+    # 1. Check if already running
+    try:
+        if requests.get(OLLAMA_HOST).status_code == 200:
+            print(" Ollama is already running.")
+            return True
+    except:
+        pass
+
+    print(" Starting Ollama Server...")
+    
+    # 2. Define Paths (CSCS Environment)
+    SCRATCH = os.environ.get("SCRATCH", "/tmp")
+    BASE_DIR = Path(SCRATCH)
+    OLLAMA_BIN = BASE_DIR / "ollama_core/bin/ollama"
+    MODELS_DIR = BASE_DIR / "ollama_core/models"
+    
+    # 3. Setup Environment
+    server_env = os.environ.copy()
+    server_env["OLLAMA_HOST"] = f"127.0.0.1:{OLLAMA_PORT}"
+    server_env["OLLAMA_MODELS"] = str(MODELS_DIR)
+    
+    # 4. Start Background Process
+    try:
+        subprocess.Popen(
+            [str(OLLAMA_BIN), "serve"], 
+            stdout=subprocess.DEVNULL, 
+            stderr=subprocess.DEVNULL, 
+            env=server_env
+        )
+        print(" Waiting for server to boot...")
+        time.sleep(10) # Give it time to initialize
+        
+        # 5. Verify
+        if requests.get(OLLAMA_HOST).status_code == 200:
+            print(" Server started successfully.")
+            return True
+    except Exception as e:
+        print(f" Failed to start server: {e}")
+        return False
+
+import requests
+import json
+import sys
+
+def pull_embedding_model(model_name="nomic-embed-text"):
+    url = "http://127.0.0.1:25000/api/pull"
+    print(f"  Requesting pull for '{model_name}'...")
+    
+    try:
+        # Send the pull request to the running server
+        response = requests.post(url, json={"name": model_name}, stream=True)
+        response.raise_for_status()
+        
+        # Stream the progress so you know it's working
+        for line in response.iter_lines():
+            if line:
+                data = json.loads(line)
+                status = data.get('status', '')
+                completed = data.get('completed', 0)
+                total = data.get('total', 1)
+                
+                # Print progress bar or status
+                if total > 1 and completed > 0:
+                    percent = int((completed / total) * 100)
+                    sys.stdout.write(f"\r   {status}: {percent}%")
+                else:
+                    sys.stdout.write(f"\r   {status}")
+                sys.stdout.flush()
+                
+        print(f"\n Model '{model_name}' installed successfully!")
+        
+    except Exception as e:
+        print(f"\n Failed to pull model: {e}")

src/transform/config.py

@@ -0,0 +1,64 @@
+import os
+import shutil
+from pathlib import Path
+from dataclasses import dataclass
+
+@dataclass
+class ProcessingConfig:
+    """Central configuration for Data Sources."""
+    root_dir: str
+    source_name: str  # e.g., 'knbs' or 'cbk'
+
+    # Settings
+    batch_size: int = 20
+    max_workers: int = 4
+    min_image_bytes: int = 3000
+    min_image_dim: int = 100
+    max_page_objects: int = 500
+
+    def __post_init__(self):
+        # Paths setup
+        self.base_processed_dir = os.path.join(self.root_dir, 'processed')
+        self.source_dir = os.path.join(self.base_processed_dir, self.source_name)
+        self.drive_zip_dir = os.path.join(self.source_dir, "zipped_batches")
+        self.meta_dir = os.path.join(self.source_dir, f"{self.source_name}_index_metadata")
+        
+        # Log Files
+        self.logs = {
+            'docs': os.path.join(self.meta_dir, f'{self.source_name}_docs_metadata.jsonl'),
+            'images': os.path.join(self.meta_dir, f'{self.source_name}_images_index.jsonl'),
+            'tables': os.path.join(self.meta_dir, f'{self.source_name}_tables_index.jsonl')
+        }
+        
+        # Local Temp Paths
+        self.local_work_dir = Path(f"/tmp/temp_work_{self.source_name}")
+        self.local_dirs = {
+            'texts': self.local_work_dir / "texts",
+            'images': self.local_work_dir / "images",
+            'tables': self.local_work_dir / "tables",
+            'pdfs': self.local_work_dir / "pdfs"
+        }
+
+    def setup(self):
+        os.makedirs(self.drive_zip_dir, exist_ok=True)
+        os.makedirs(self.meta_dir, exist_ok=True)
+        if self.local_work_dir.exists():
+            shutil.rmtree(self.local_work_dir)
+        for d in self.local_dirs.values():
+            d.mkdir(parents=True, exist_ok=True)
+        self.create_canary()
+
+    def create_canary(self):
+        script_content = """
+import sys, pymupdf, pdfplumber
+if len(sys.argv) < 2: sys.exit(1)
+try:
+    doc = pymupdf.open(sys.argv[1])
+    for p in doc: _, _ = p.get_text(), [doc.extract_image(i[0]) for i in p.get_images(full=True)]
+    with pdfplumber.open(sys.argv[1]) as p: _ = [page.objects for page in p.pages]
+    print("SAFE")
+    sys.exit(0)
+except: sys.exit(1)
+"""
+        with open("pdf_canary.py", "w") as f:
+            f.write(script_content.strip())

src/transform/download_files.py

@@ -0,0 +1,141 @@
+import os
+import json
+import zipfile
+import time
+import gc
+import threading
+import shutil
+import requests
+import subprocess
+import pymupdf
+import pdfplumber
+import pandas as pd
+import urllib3
+from pathlib import Path
+from datetime import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+# Import shared config
+from config import ProcessingConfig
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class UniversalProcessor:
+    def __init__(self, config: ProcessingConfig):
+        self.config = config
+        retry = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 504])
+        adapter = HTTPAdapter(max_retries=retry)
+        self.session = requests.Session()
+        self.session.mount("https://", adapter)
+        self.session.mount("http://", adapter)
+
+    def download(self, url, safe_title) -> Path:
+        try:
+            response = self.session.get(url, timeout=30, stream=True, verify=False)
+            response.raise_for_status()
+            ext = Path(url).suffix.lower() or '.pdf'
+            safe_name = safe_title[:50].replace(' ', '_').replace('/', '_')
+            filepath = self.config.local_dirs['pdfs'] / f"{safe_name}{ext}"
+            with open(filepath, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            return filepath
+        except: return None
+
+    def process(self, filepath: Path, url: str, safe_title: str):
+        # Only does basic validation/extraction for the initial pass
+        try:
+            res = subprocess.run(["python", "pdf_canary.py", str(filepath)], capture_output=True, timeout=15)
+            if res.returncode != 0: return None
+            
+            # Basic PyMuPDF extraction for quick preview
+            doc = pymupdf.open(filepath)
+            text = "".join([page.get_text() for page in doc])
+            doc.close()
+            
+            return {
+                'text': text,
+                'tables': [],
+                'images': [],
+                'metadata': {'pages': len(doc)}
+            }
+        except: return None
+
+class BatchPipeline:
+    def __init__(self, config: ProcessingConfig, processor: UniversalProcessor):
+        self.config = config
+        self.processor = processor
+        self.lock = threading.Lock()
+        self.config.setup()
+
+    def _append_log(self, log_key, record):
+        with self.lock:
+            with open(self.config.logs[log_key], 'a', encoding='utf-8') as f:
+                f.write(json.dumps(record) + '\n')
+
+    def _worker(self, item):
+        row = item['row']
+        title = str(row.get('text', 'untitled'))
+        url = row['file_url']
+        
+        path = self.processor.download(url, title)
+        if not path or path.stat().st_size < 500: return None
+        
+        data = self.processor.process(path, url, title)
+        if not data: return None
+        
+        # Save Preview Text
+        with open(self.config.local_dirs['texts'] / f"{path.stem}.txt", 'w') as f:
+            f.write(data['text'])
+            
+        self._append_log('docs', {'url': url, 'file': path.name, 'status': 'downloaded'})
+        return True
+
+    def _zip_and_ship(self, batch_id):
+        ts = datetime.now().strftime("%H%M%S")
+        zname = f"{self.config.source_name}_{batch_id}_{ts}.zip"
+        local_z = Path(f"/tmp/{zname}")
+        drive_z = Path(self.config.drive_zip_dir) / zname
+        
+        with zipfile.ZipFile(local_z, 'w', zipfile.ZIP_DEFLATED) as z:
+            for root, _, files in os.walk(self.config.local_work_dir):
+                for f in files:
+                    fp = os.path.join(root, f)
+                    z.write(fp, os.path.relpath(fp, self.config.local_work_dir))
+        
+        shutil.copy(local_z, drive_z)
+        self.config.setup() # Wipe local
+        os.remove(local_z)
+
+    def run(self, df, ignore_history=False):
+        done = set()
+        if not ignore_history and os.path.exists(self.config.logs['docs']):
+            with open(self.config.logs['docs']) as f:
+                for l in f: 
+                    try: done.add(json.loads(l)['url'])
+                    except: continue
+        
+        queue = [r for _, r in df.iterrows() if r['file_url'] not in done]
+        print(f" Queued: {len(queue)} files.")
+        
+        bs = self.config.batch_size
+        for i in range(0, len(queue), bs):
+            batch = queue[i:i+bs]
+            bid = f"batch_{i//bs + 1}"
+            print(f"Processing {bid}...")
+            
+            with ThreadPoolExecutor(max_workers=self.config.max_workers) as ex:
+                futures = [ex.submit(self._worker, {'row': item}) for item in batch]
+                for _ in tqdm(as_completed(futures), total=len(batch)): pass
+            
+            self._zip_and_ship(bid)
+            gc.collect()
+
+if __name__ == "__main__":
+    # Example Usage
+    ROOT_DIR = "/scratch/user/mshauri_data"
+    conf = ProcessingConfig(root_dir=ROOT_DIR, source_name='cbk')
+    pipe = BatchPipeline(conf, UniversalProcessor(conf))

src/transform/extract.py

@@ -0,0 +1,279 @@
+import random
+
+import os
+import sys
+import time
+import queue
+import logging
+import gc
+import multiprocessing as mp
+import argparse
+from pathlib import Path
+import torch
+
+def configure_parallelism(workers_per_gpu=5):
+    """
+    Tuned specifically for 96GB VRAM GPUs.
+    We cap workers to prevent 'Thundering Herd' and use VRAM for 'Batch Power'.
+    """
+    if not torch.cuda.is_available():
+        return max(1, mp.cpu_count() // 2), 1, 0
+
+    num_gpus = torch.cuda.device_count()
+    gpu_properties = torch.cuda.get_device_properties(0)
+    total_vram_gb = gpu_properties.total_memory / (1024**3)
+    
+    # --- THE STABILITY STRATEGY ---
+    # On 96GB, 8-10 workers is the "Sweet Spot". 
+    # More workers than this creates too much 'context switching' overhead on the GPU. 
+    
+    total_slots = num_gpus * workers_per_gpu
+    
+    print(f"🔍 GH200/A100 Detected: {num_gpus} GPUs | {total_vram_gb:.1f} GB VRAM")
+    print(f"⚙️  Stability Config: {workers_per_gpu} workers/GPU | {total_slots} Total Slots")
+    
+    # --- SYSTEM TUNING ---
+    os.environ["OLLAMA_NUM_PARALLEL"] = str(total_slots)
+    os.environ["OLLAMA_MAX_QUEUE"] = "2048" # Large buffer for requests
+    os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+    
+    return total_slots, workers_per_gpu, num_gpus
+
+
+
+# --- CRITICAL: MULTIPROCESSING SETUP ---
+try:
+    mp.set_start_method('spawn', force=True)
+except RuntimeError:
+    pass 
+
+from marker.converters.pdf import PdfConverter
+from marker.models import create_model_dict
+from marker.output import text_from_rendered
+from marker.config.parser import ConfigParser
+
+# Configure Logger
+logging.basicConfig(
+    level=logging.INFO, 
+    format='%(asctime)s - [GPU-%(processName)s] - %(message)s',
+    datefmt='%H:%M:%S'
+)
+logger = logging.getLogger(__name__)
+
+def worker_routine(worker_id, gpu_id, batch_queue, output_dir, ollama_config, marker_config):
+    """
+    Optimized Worker for GH200:
+    1. Receives a BATCH of files (List) to reduce queue overhead.
+    2. Uses torch.compile for architectural optimization.
+    3. Skips image extraction for speed.
+    4. Fails fast on tables (Triage) to prevent LLM stalls.
+    """
+    
+    time.sleep(worker_id * 1.5) 
+    
+    mp.current_process().name = f"{worker_id}:Dev{gpu_id}"
+    logger.info(f"Initializing Worker {worker_id}...")
+    
+    os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
+    os.environ["TORCH_DEVICE"] = f"cuda:{gpu_id}"
+    
+    # 2. Model Initialization
+    try:
+        # Load Surya/OCR weights
+        artifact_dict = create_model_dict(
+            device=f"cuda:{gpu_id}", 
+            dtype=torch.bfloat16, 
+            attention_implementation="flash_attention_2"
+        )
+
+        # --- OPTIMIZATION 1: torch.compile ---
+        logger.info("Compiling models with torch.compile... (One-time setup)")
+        for key, model in artifact_dict.items():
+            if hasattr(model, 'forward'):
+                artifact_dict[key] = torch.compile(model, mode="max-autotune")
+
+        # --- OPTIMIZATION 2: Config Tuning ---
+        full_config = {
+            "output_format": "markdown",
+            "disable_multiprocessing": True,
+            "extract_images": False,         # Speed up: Skip image extraction
+            "ocr_all_pages": False,
+            "use_llm": True,
+            "llm_service": "marker.services.ollama.OllamaService",
+            
+            # --- TRIAGE STRATEGY ---
+            "max_table_retries": 0,          # Fail fast if table extraction stalls
+            "llm_service_timeout": 150, # Don't let a table hold a worker for more than x minutes
+            
+            **ollama_config,
+            **marker_config
+        }
+        
+        config_parser = ConfigParser(full_config)
+        
+        converter = PdfConverter(
+            config=config_parser.generate_config_dict(),
+            artifact_dict=artifact_dict,
+            processor_list=config_parser.get_processors(),
+            renderer=config_parser.get_renderer(),
+            llm_service=config_parser.get_llm_service()
+        )
+        logger.info(f"Worker Ready. Waiting for batches...")
+        
+    except Exception as e:
+        logger.error(f"Initialization Failed: {e}")
+        return
+
+    # 3. Batch Work Loop
+    batches_processed = 0
+    while True:
+        try:
+            # Get a list of files (Batch)
+            batch_files = batch_queue.get(timeout=5)
+        except queue.Empty:
+            logger.info(f"Queue empty. Worker shutting down. Processed {batches_processed} batches.")
+            break
+            
+        # Process the batch locally
+        for pdf_path_str in batch_files:
+            try:
+                pdf_path = Path(pdf_path_str)
+                doc_out_dir = Path(output_dir) / pdf_path.stem
+                md_file = doc_out_dir / f"{pdf_path.stem}.md"
+
+                if md_file.exists():
+                    continue
+
+                # Heavy Compute (OCR + LLM)
+                rendered = converter(str(pdf_path))
+                
+                if rendered is None:
+                    logger.warning(f"Skipping {pdf_path.name}: Converter returned None")
+                    continue
+
+                text, meta, images = text_from_rendered(rendered)
+
+                # Write Output
+                doc_out_dir.mkdir(parents=True, exist_ok=True)
+                with open(md_file, "w", encoding="utf-8") as f:
+                    f.write(text)
+            
+            except Exception as e:
+                logger.error(f"Failed on {pdf_path.name}: {e}")
+        
+        # Cleanup after batch to keep VRAM healthy
+        batches_processed += 1
+        
+        # Aggressive GC after every batch prevents "memory creep" on long runs
+        gc.collect()
+        torch.cuda.empty_cache()
+
+class MarkerFolderProcessor:
+    def __init__(self, output_dir, ollama_url, ollama_model, batch_multiplier, workers_per_gpu, num_gpus):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.num_gpus = num_gpus
+        
+        # We now accept the dynamic num_gpus passed from __main__
+        if self.num_gpus > 0:
+            print(f" Detected {self.num_gpus} GPUs (Dynamic Mode)")
+        else:
+            print(" No GPUs detected. Running in CPU mode.")
+
+        self.workers_per_gpu = workers_per_gpu
+        
+        # Configs passed to workers
+        self.ollama_config = {
+            "ollama_base_url": ollama_url,
+            "ollama_model": ollama_model,
+            "ollama_timeout": 600, # 10 mins max per request
+            "ollama_options": {
+                "num_ctx": 32768,
+                "num_predict": 2048,
+                "temperature": 0.0
+            }
+        }
+        self.marker_config = {"batch_multiplier": batch_multiplier}
+
+    def process_folder(self, source_folder, batch_size=10, subset=None):
+        if subset is not None:
+            # Use the partitioned list provided by run_transform.py
+            pdfs = [Path(p) for p in subset]
+        else:
+            source_path = Path(source_folder)
+            pdfs = sorted(list(source_path.glob("*.pdf")))
+        
+        if not pdfs:
+            print("No PDFs to process.")
+            return
+
+        manager = mp.Manager()
+        batch_queue = manager.Queue()
+        
+        # --- BATCHING STRATEGY ---
+        # Chunk the list of PDFs into batches
+        chunks = [pdfs[i:i + batch_size] for i in range(0, len(pdfs), batch_size)]
+        print(f" Created {len(chunks)} batches of {batch_size} files each.")
+        
+        for chunk in chunks:
+            batch_queue.put([str(p) for p in chunk])
+
+        total_workers = (self.num_gpus * self.workers_per_gpu) if self.num_gpus > 0 else 1
+        print(f" Launching {total_workers} workers on {self.num_gpus} GPUs...")
+
+        processes = []
+        for i in range(total_workers):
+            gpu_id = i % self.num_gpus if self.num_gpus > 0 else 0
+            p = mp.Process(
+                target=worker_routine,
+                args=(i, gpu_id, batch_queue, self.output_dir, self.ollama_config, self.marker_config)
+            )
+            p.start()
+            processes.append(p)
+
+        for p in processes:
+            p.join()
+            
+        print(" Extraction Complete.")
+
+# This block only runs if you execute 'python extract.py' directly.
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input", required=True, help="Input folder of PDFs")
+    parser.add_argument("--output", required=True, help="Output folder")
+    parser.add_argument("--url", default="http://localhost:11434", help="Ollama URL")
+    parser.add_argument("--model", default="llama3", help="Ollama Model Name")
+    args = parser.parse_args()
+
+    # --- DYNAMIC HARDWARE DETECTION ---
+    if torch.cuda.is_available():
+        num_gpus = torch.cuda.device_count()
+        gpu_properties = torch.cuda.get_device_properties(0)
+        total_vram_gb = gpu_properties.total_memory / (1024**3)
+        
+        # Calculate optimal workers: (VRAM - 2GB overhead) / 5GB per worker
+        workers_per_gpu = int((total_vram_gb - 2) // 5)
+        workers_per_gpu = max(1, workers_per_gpu) # Minimum 1
+        
+        total_slots = num_gpus * workers_per_gpu
+        print(f"⚙️  Dynamic Config: {num_gpus} GPUs | {workers_per_gpu} workers/GPU | {total_slots} Total Slots")
+        
+        # Set Env vars for external tools (optional, but good practice)
+        os.environ["OLLAMA_NUM_PARALLEL"] = str(total_slots)
+        
+    else:
+        num_gpus = 0
+        workers_per_gpu = 1
+        print("  No GPU detected. Defaulting to 1 worker.")
+
+    # --- PROCESSOR INIT ---
+    processor = MarkerFolderProcessor(
+        output_dir=args.output,
+        ollama_url=args.url,
+        ollama_model=args.model,
+        batch_multiplier=2,      
+        workers_per_gpu=workers_per_gpu, # Passed dynamically
+        num_gpus=num_gpus                # Passed dynamically
+    )
+    
+    processor.process_folder(args.input, batch_size=10)

src/transform/get_csv_from_md.py

@@ -0,0 +1,223 @@
+import os
+import re
+import json
+import io
+import time
+import logging
+import requests
+import subprocess
+import pandas as pd
+from pathlib import Path
+from sqlalchemy import create_engine
+from ollama import Client
+
+# --- LOGGING ---
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
+logger = logging.getLogger("KNBS_Ingest")
+
+# --- 1. INFRASTRUCTURE ---
+
+def _manage_ollama_server(ollama_host, ollama_port, ollama_bin, model):
+    """Ensures Ollama is running (reuses existing logic)."""
+    try:
+        if requests.get(ollama_host).status_code == 200:
+            logger.info(" Ollama connected.")
+            return True
+    except: pass
+
+    logger.info(f"🚀 Starting Ollama ({model})...")
+    scratch_env = os.environ.get("SCRATCH", "/tmp")
+    models_dir = Path(scratch_env) / "ollama_core/models"
+    
+    server_env = os.environ.copy()
+    server_env["OLLAMA_HOST"] = f"127.0.0.1:{ollama_port}"
+    server_env["OLLAMA_MODELS"] = str(models_dir)
+    
+    try:
+        subprocess.Popen([str(ollama_bin), "serve"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=server_env)
+        time.sleep(5)
+        subprocess.run([str(ollama_bin), "pull", model], env=server_env, check=True)
+        return True
+    except Exception as e:
+        logger.error(f" Server Error: {e}")
+        return False
+
+# --- 2. MARKDOWN PARSING ENGINE ---
+
+def extract_tables_from_markdown(md_content: str) -> list[pd.DataFrame]:
+    """
+    Scans markdown text for pipe-delimited tables (| col | col |) 
+    and converts them to Pandas DataFrames.
+    """
+    tables = []
+    lines = md_content.split('\n')
+    buffer = []
+    inside_table = False
+    
+    for line in lines:
+        stripped = line.strip()
+        # Detect table lines (must start and end with |)
+        if stripped.startswith('|') and stripped.endswith('|'):
+            inside_table = True
+            buffer.append(stripped)
+        else:
+            if inside_table:
+                # Table block ended, process buffer
+                if buffer:
+                    try:
+                        table_str = '\n'.join(buffer)
+                        # Read using pandas, handling markdown separators
+                        df = pd.read_csv(
+                            io.StringIO(table_str), 
+                            sep="|", 
+                            skipinitialspace=True, 
+                            engine='python'
+                        )
+                        
+                        # CLEANUP PANDAS ARTIFACTS
+                        # 1. Drop empty columns (pandas creates empty cols for leading/trailing pipes)
+                        df = df.dropna(axis=1, how='all')
+                        
+                        # 2. Filter out the markdown divider row (e.g. ---|---|---)
+                        if not df.empty:
+                            df = df[~df.iloc[:,0].astype(str).str.contains('---', regex=False)]
+                            
+                        if not df.empty and len(df.columns) > 1:
+                            tables.append(df)
+                            
+                    except Exception as e:
+                        logger.warning(f"Failed to parse a table block: {e}")
+                
+                buffer = []
+                inside_table = False
+                
+    return tables
+
+# --- 3. LLM HEADER CLEANER (KNBS SPECIFIC) ---
+
+def clean_knbs_headers(df: pd.DataFrame, filename: str, table_index: int, client: Client, model: str) -> pd.DataFrame:
+    """
+    Uses LLM to sanitize headers, handling split headers common in PDF-to-Markdown.
+    """
+    raw_headers = [str(c).strip() for c in df.columns]
+    
+    # Context: Provide first 2 rows to help identify if headers are split across rows
+    data_preview = df.head(2).astype(str).values.tolist()
+    
+    prompt = f"""
+    You are a Data Engineer cleaning Kenya National Bureau of Statistics (KNBS) data.
+    
+    Source File: "{filename}"
+    Table Index: {table_index}
+    
+    Current Headers: {raw_headers}
+    Data Preview (First 2 Rows): {data_preview}
+    
+    Task: Return a list of {len(raw_headers)} clean, snake_case SQL column names.
+    
+    RULES:
+    1. INFER MEANING: If header is "Gross" and Row 1 is "Domestic Product", the column name is "gdp".
+    2. HANDLE YEARS: If headers are "2019", "2020", keep as "year_2019".
+    3. HANDLE GARBAGE: If header is "Unnamed: 1" look at Data Preview. If it contains items like "Agriculture", name it "sector".
+    4. KNBS reports often have a "Total" column. Ensure it is named "total".
+    
+    Respond ONLY with a JSON list of strings.
+    """
+    
+    try:
+        res = client.chat(model=model, messages=[{'role': 'user', 'content': prompt}], format='json')
+        new_headers = json.loads(res['message']['content'])
+        
+        # Handle dictionary wrapper if LLM returns {"headers": [...]}
+        if isinstance(new_headers, dict):
+            for val in new_headers.values():
+                if isinstance(val, list): 
+                    new_headers = val
+                    break
+        
+        # Validation: Length must match
+        if isinstance(new_headers, list) and len(new_headers) == len(df.columns):
+            df.columns = new_headers
+        else:
+            # Fallback: keep originals but snake_case them
+            df.columns = [re.sub(r'[^a-zA-Z0-9]', '_', str(c).strip()).lower() for c in df.columns]
+            
+    except Exception as e:
+        logger.warning(f"LLM Header clean failed (Table {table_index}): {e}")
+        
+    return df
+
+# --- 4. MAIN PIPELINE EXPORT ---
+
+def ingest_knbs_data(input_dir: str, db_name: str, model: str = "qwen2.5:14b"):
+    """
+    Main entry point to run the KNBS ingestion pipeline.
+    Recursively scans input_dir for all .md files.
+    """
+    # Paths
+    SCRATCH = os.environ.get("SCRATCH", "/tmp")
+    BASE_DIR = Path(SCRATCH)
+    
+    INPUT_PATH = Path(input_dir)
+    if not INPUT_PATH.exists():
+        INPUT_PATH = BASE_DIR / input_dir
+    
+    if not INPUT_PATH.exists():
+        logger.error(f" Input directory not found: {INPUT_PATH}")
+        return
+
+    OLLAMA_BIN = BASE_DIR / "ollama_core/bin/ollama"
+    CUSTOM_PORT = "25000"
+    OLLAMA_HOST = f"http://127.0.0.1:{CUSTOM_PORT}"
+    
+    # Infrastructure
+    if not _manage_ollama_server(OLLAMA_HOST, CUSTOM_PORT, OLLAMA_BIN, model): return
+    
+    engine = create_engine(f"sqlite:///{db_name}")
+    client = Client(host=OLLAMA_HOST)
+    
+    # Process Files (RECURSIVE SEARCH using rglob)
+    files = sorted(list(INPUT_PATH.rglob("*.md")))
+    logger.info(f"🚀 Found {len(files)} KNBS markdown files (Recursive Scan). Starting ingestion...")
+    
+    for f in files:
+        logger.info(f"📄 Processing {f.name}...")
+        try:
+            with open(f, 'r', encoding='utf-8', errors='ignore') as file:
+                content = file.read()
+            
+            # A. Extract Tables
+            dfs = extract_tables_from_markdown(content)
+            
+            if not dfs:
+                continue
+                
+            logger.info(f"   found {len(dfs)} tables.")
+            
+            # B. Clean & Load Tables
+            for i, df in enumerate(dfs):
+                # Basic cleanup
+                df = df.dropna(how='all', axis=1).dropna(how='all', axis=0)
+                if df.empty or len(df) < 2: continue # Skip empty/tiny tables
+                
+                # LLM Semantic Cleaning
+                df = clean_knbs_headers(df, f.name, i, client, model)
+                
+                # Sanitize numeric data
+                for c in df.columns:
+                    if any(x in str(c).lower() for x in ['rate', 'value', 'amount', 'total', 'year', 'price']):
+                        df[c] = df[c].apply(lambda x: pd.to_numeric(str(x).replace(',', '').replace('%',''), errors='ignore'))
+
+                # Naming: knbs_{filename_slug}_tab{index}
+                slug = re.sub(r'[^a-zA-Z0-9]', '_', f.stem).lower()[:40].lstrip('_')
+                table_name = f"{slug}_tab{i}"
+                
+                df['source_file'] = f.name
+                
+                df.to_sql(table_name, engine, if_exists='replace', index=False)
+                logger.info(f"     -> Saved table: {table_name} ({len(df)} rows)")
+                
+        except Exception as e:
+            logger.error(f"   Failed {f.name}: {e}")
+
+    logger.info(" KNBS Ingestion Complete.")

src/transform/run_transform.py

@@ -0,0 +1,133 @@
+import sys
+import os
+import subprocess
+import time
+import logging
+import requests
+import torch
+from pathlib import Path
+from datetime import timedelta
+
+# --- 1. LOGGING SETUP ---
+# Identify Node Rank for logging clarity
+NODE_ID = os.environ.get("SLURM_PROCID", "0")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format=f'%(asctime)s - [Node {NODE_ID}] - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(f"logs/node_{NODE_ID}_transform.log")
+    ]
+)
+logger = logging.getLogger(__name__)
+
+def main():
+    t_start = time.perf_counter()
+    logger.info(f"🚀 Starting Transformation Pipeline on Node {NODE_ID}")
+
+    # --- 2. ENVIRONMENT & PATHS ---
+    SCRATCH = Path(os.environ.get("SCRATCH", "/tmp"))
+    INPUT_PDFS_DIR = SCRATCH / "mshauri-fedha/data/knbs/pdfs"
+    OUTPUT_DIR = SCRATCH / "mshauri-fedha/data/knbs/marker-output"
+    
+    OLLAMA_HOME = SCRATCH / "ollama_core"
+    OLLAMA_BIN = OLLAMA_HOME / "bin/ollama"
+    OLLAMA_HOST = "http://localhost:11434"
+
+    # Important: Ensure the current directory is in sys.path for 'extract' import
+    if os.getcwd() not in sys.path:
+        sys.path.append(os.getcwd())
+
+    try:
+        from extract import MarkerFolderProcessor, configure_parallelism
+    except ImportError as e:
+        logger.error(f"Could not import extract.py from {os.getcwd()}")
+        raise e
+
+    # --- 3. DYNAMIC PARALLELISM & OLLAMA CONFIG ---
+    # Calculates workers based on node hardware (GH200 96GB)
+    total_slots, workers_per_gpu, num_gpus = configure_parallelism()
+    
+    # Clean up any zombie servers on this node
+    subprocess.run(["pkill", "-f", "ollama serve"], stderr=subprocess.DEVNULL)
+    time.sleep(5)
+
+    # Set server environment variables
+    server_env = os.environ.copy()
+    server_env["OLLAMA_NUM_PARALLEL"] = str(total_slots)
+    server_env["OLLAMA_MAX_LOADED_MODELS"] = "1"
+    server_env["OLLAMA_MAX_QUEUE"] = "2048"
+    server_env["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+    logger.info(f"⏳ Launching Ollama Server with {total_slots} slots...")
+    subprocess.Popen(
+        [str(OLLAMA_BIN), "serve"], 
+        stdout=subprocess.DEVNULL, 
+        stderr=subprocess.DEVNULL,
+        env=server_env
+    )
+
+    # Heartbeat check
+    for i in range(60):
+        try:
+            if requests.get(OLLAMA_HOST).status_code == 200:
+                logger.info(" Ollama Server is online.")
+                break
+        except:
+            time.sleep(1)
+    else:
+        raise RuntimeError(" Ollama server heartbeat failed.")
+
+    # --- 4. MODEL SETUP ---
+    BASE_MODEL = "qwen2.5:7b" 
+    CUSTOM_MODEL_NAME = "qwen2.5-7b-16k"
+
+    logger.info(f" Pulling {BASE_MODEL}...")
+    subprocess.run([str(OLLAMA_BIN), "pull", BASE_MODEL], check=True, capture_output=True)
+
+    logger.info(f" Creating custom 16k context model...")
+    modelfile_path = Path(f"Modelfile_node_{NODE_ID}")
+    modelfile_path.write_text(f"FROM {BASE_MODEL}\nPARAMETER num_ctx 16384")
+    subprocess.run([str(OLLAMA_BIN), "create", CUSTOM_MODEL_NAME, "-f", str(modelfile_path)], check=True, capture_output=True)
+
+    # --- 5. AUTOMATED DATA PARTITIONING ---
+    # Get all PDFs and sort them for deterministic behavior
+    all_pdfs = sorted(list(INPUT_PDFS_DIR.glob("*.pdf")))
+    total_nodes = int(os.environ.get("SLURM_NTASKS", 1))
+    node_rank = int(NODE_ID)
+
+    # Each node takes every Nth file (Node 0 takes index 0, 2, 4... Node 1 takes 1, 3, 5...)
+    my_pdfs = all_pdfs[node_rank::total_nodes]
+    my_pdf_strs = [str(p) for p in my_pdfs]
+
+    logger.info(f" Data Partitioning: Node {node_rank}/{total_nodes} handling {len(my_pdfs)} files.")
+
+    # --- 6. EXECUTION ---
+    os.chdir(SCRATCH)
+    
+    processor = MarkerFolderProcessor(
+        output_dir=OUTPUT_DIR,
+        ollama_url=OLLAMA_HOST,
+        ollama_model=CUSTOM_MODEL_NAME,
+        batch_multiplier=4, 
+        workers_per_gpu=workers_per_gpu,
+        num_gpus=num_gpus                   
+    )
+
+    logger.info(f"🚀 Processing PDFs...")
+    # Using the 'subset' parameter in process_folder (ensure extract.py supports this)
+    processor.process_folder(INPUT_PDFS_DIR, batch_size=5, subset=my_pdf_strs)
+
+    # --- 7. CLEANUP & TIMING ---
+    t_end = time.perf_counter()
+    duration = timedelta(seconds=t_end - t_start)
+    logger.info(" Transformation process finished.")
+    logger.info(f"⏱️ Total Duration for Node {NODE_ID}: {duration}")
+    
+    # Shutdown server
+    subprocess.run(["pkill", "-f", "ollama serve"], stderr=subprocess.DEVNULL)
+    if modelfile_path.exists(): modelfile_path.unlink()
+
+if __name__ == "__main__":
+    main()

src/transform/structure_data.py

@@ -0,0 +1,322 @@
+import os
+import re
+import json
+import time
+import logging
+import requests
+import subprocess
+import pandas as pd
+import numpy as np
+from typing import List, Tuple, Dict
+from pathlib import Path
+from sqlalchemy import create_engine, text
+from ollama import Client
+
+# --- LOGGING ---
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
+logger = logging.getLogger("SmartIngestV6")
+
+# --- 0. KNOWLEDGE BASE (THE RULES) ---
+
+# Files to ignore completely 
+SKIP_PATTERNS = [
+    "december__2019_tap",
+    "lcr_return",
+    "lcr_sheet",
+    "quarterly_gdp",
+    "remittances",
+    "depository_corporation_survey_(expanded)"
+]
+
+# Exact column expectations based on prior analysis [cite: 1, 2, 5, 6, 8, 14, 15, 16, 17]
+SCHEMA_DEFINITIONS = {
+    "annual_gdp": ["year", "month", "nominal_gdp_prices", "real_gdp_growth", "real_gdp_prices"],
+    "bop_annual": ["bpm6_concept", "year_2019", "year_2020", "year_2021", "year_2022", "year_2023", "year_2024"],
+    "indicative_rates": ["date", "currency", "mean_rate", "buy_rate", "sell_rate"],
+    "exchange_rates": ["date", "currency", "mean_rate", "buy_rate", "sell_rate"], # Catch-all for historical/indicative
+    "central_bank_rates": ["year", "month", "reverse_repo", "interbank_rate", "tbill_91_day", "tbill_182_day", "tbill_364_day", "reserve_requirement", "cbr"],
+    "commercial_bank_rates": ["year", "month", "deposit_rate", "savings_rate", "lending_rate", "overdraft_rate"],
+    "domestic_debt": ["fiscal_year", "treasury_bills", "treasury_bonds", "govt_stocks", "overdraft_cbk", "advances_commercial", "other_debt", "total_debt"],
+    "forex_bureau": ["bureau_name", "usd_buy", "usd_sell", "usd_margin", "gbp_buy", "gbp_sell", "gbp_margin", "euro_buy", "euro_sell", "euro_margin"],
+    "treasury_bills": ["issue_date", "amount_offered", "tenure", "amount_received", "amount_accepted", "yield_rate", "alloted", "rejected", "redeemed", "outstanding"],
+    "treasury_bonds": ["issue_date", "bond_code", "amount_offered", "amount_received", "amount_accepted", "coupon_rate", "alloted", "rejected", "redeemed", "outstanding"],
+    "exports": ["year", "month", "commodity", "value_millions", "total"],
+    "imports": ["year", "month", "commodity", "value_millions", "total"],
+    "revenue": ["year", "month", "tax_revenue", "non_tax_revenue", "total_revenue", "recurrent_expenditure", "development_expenditure"],
+    "depository_corporation_survey": ["category", "data_values"] # Wide table handling triggered later
+}
+
+# --- 1. INFRASTRUCTURE ---
+
+def _manage_ollama_server(ollama_host, ollama_port, ollama_bin, model):
+    try:
+        if requests.get(ollama_host).status_code == 200:
+            logger.info(" Ollama connected.")
+            return True
+    except: pass
+
+    logger.info(f" Starting Ollama ({model})...")
+    scratch_env = os.environ.get("SCRATCH", "/tmp")
+    models_dir = Path(scratch_env) / "ollama_core/models"
+    
+    server_env = os.environ.copy()
+    server_env["OLLAMA_HOST"] = f"127.0.0.1:{ollama_port}"
+    server_env["OLLAMA_MODELS"] = str(models_dir)
+    models_dir.mkdir(parents=True, exist_ok=True)
+    
+    try:
+        subprocess.Popen([str(ollama_bin), "serve"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=server_env)
+        time.sleep(5)
+        subprocess.run([str(ollama_bin), "pull", model], env=server_env, check=True)
+        return True
+    except Exception as e:
+        logger.error(f" Server Error: {e}")
+        return False
+
+# --- 2. HEADER HUNTER (Geometric Scanner) ---
+
+def read_csv_robust(file_path: Path) -> pd.DataFrame:
+    encodings = ['utf-8', 'latin1', 'cp1252', 'ISO-8859-1']
+    for enc in encodings:
+        try:
+            return pd.read_csv(file_path, header=None, dtype=str, encoding=enc).fillna("")
+        except UnicodeDecodeError:
+            continue
+    return pd.DataFrame()
+
+def find_best_header_row(df_raw: pd.DataFrame, expected_keywords: List[str]) -> Tuple[int, int]:
+    """Scores rows based on expected keywords for this specific file type."""
+    scores = {}
+    scan_depth = min(30, len(df_raw))
+    
+    # If we have no expectations, use generic keywords
+    if not expected_keywords:
+        expected_keywords = ['year', 'month', 'date', 'rate', 'bank', 'shilling', 'total']
+
+    for i in range(scan_depth):
+        row_str = " ".join(df_raw.iloc[i].astype(str)).lower()
+        score = 0
+        
+        # Reward: Matches expected schema
+        for kw in expected_keywords:
+            if kw.lower() in row_str:
+                score += 3
+        
+        # Penalty: Looks like Data (Dense numbers)
+        num_cells = sum(1 for c in df_raw.iloc[i].astype(str) if c.replace(',','').replace('.','').isdigit())
+        if num_cells > len(df_raw.columns) * 0.5:
+            score -= 10
+            
+        scores[i] = score
+
+    best_header = max(scores, key=scores.get)
+    if scores[best_header] <= 0:
+        return _geometric_scan(df_raw)
+    
+    return best_header, best_header + 1
+
+def _geometric_scan(df_raw):
+    """Fallback: Find first dense block of numbers."""
+    def is_data(x):
+        try: 
+            float(str(x).replace(',', ''))
+            return 1
+        except: return 0
+    scores = df_raw.map(is_data).sum(axis=1)
+    if scores.empty or scores.max() <= 1: return 0, 1
+    data_rows = scores[scores >= scores.max() * 0.5].index.tolist()
+    if not data_rows: return 0, 1
+    data_start = data_rows[0]
+    header_idx = max(0, data_start - 1)
+    # Search up for content
+    while header_idx > 0:
+        if df_raw.iloc[header_idx].str.join("").str.strip().any(): break
+        header_idx -= 1
+    return header_idx, data_start
+
+# --- 3. HYBRID PROMPT STRATEGY ---
+
+def get_clean_headers(raw_headers: List[str], first_row: List[str], filename: str, client: Client, model: str) -> List[str]:
+    # 1. Identify File Type & Expectations
+    expected_cols = []
+    file_type = "generic"
+    for key, cols in SCHEMA_DEFINITIONS.items():
+        if key in filename.lower():
+            file_type = key
+            expected_cols = cols
+            break
+            
+    # 2. Build Prompt
+    valid_raw = [str(h).strip() for h in raw_headers]
+    valid_data = [str(d).strip()[:15] for d in first_row]
+    
+    prompt = f"""
+    You are a Financial Data Engineer.
+    
+    File: "{filename}"
+    Detected Type: "{file_type}"
+    Expected Schema: {expected_cols}
+    
+    Current Headers (Row N): {valid_raw}
+    First Data Row (Row N+1): {valid_data}
+    
+    Task: Return a list of {len(raw_headers)} clean snake_case column names.
+    
+    CRITICAL RULES:
+    1. PRIORITIZE THE EXPECTED SCHEMA. If the data looks like it matches the expectation, use those names.
+    2. If Expected Schema has 5 cols but file has 7, keep the 5 and name the others based on context (e.g., 'total').
+    3. If header is a Year ("1999"), keep it as "year_1999".
+    4. If header is empty/garbage, use the Data Row to guess (e.g. "Kenya Commercial Bank" -> "bank_name").
+    
+    Respond ONLY with a JSON list of strings.
+    """
+    
+    try:
+        res = client.chat(model=model, messages=[{'role': 'user', 'content': prompt}], format='json')
+        content = json.loads(res['message']['content'])
+        
+        if isinstance(content, dict):
+            for val in content.values():
+                if isinstance(val, list): return val
+        return content if isinstance(content, list) else [f"col_{i}" for i in range(len(raw_headers))]
+    except:
+        # FALLBACK: If LLM fails, return the Expected Schema (padded if needed)
+        if expected_cols:
+            if len(expected_cols) < len(raw_headers):
+                return expected_cols + [f"extra_{i}" for i in range(len(raw_headers)-len(expected_cols))]
+            return expected_cols[:len(raw_headers)]
+        return [f"col_{i}" for i in range(len(raw_headers))]
+
+# --- 4. SPECIFIC TRANSFORMS ---
+
+def apply_specific_transforms(df: pd.DataFrame, filename: str) -> pd.DataFrame:
+    fname = filename.lower()
+    
+    # Rule 20: Revenue & Expenditure - Remove top 3 rows
+    if "revenue" in fname:
+        if len(df) > 3: df = df.iloc[3:].reset_index(drop=True)
+            
+    # Rule 9: Depository Survey - Wide Table Logic
+    if "depository_corporation" in fname:
+        # This is a massive wide table. We usually want to melt it.
+        # Assuming col 0 is Category and rest are dates
+        try:
+            id_vars = [df.columns[0]]
+            value_vars = [c for c in df.columns if c != df.columns[0]]
+            df = df.melt(id_vars=id_vars, value_vars=value_vars, var_name="date", value_name="amount_millions")
+        except: pass
+
+    # Rule 1/19/21/22: Year + Month merging
+    # Check if we have 'year' and 'month' columns
+    cols = [str(c).lower() for c in df.columns]
+    if 'year' in cols and 'month' in cols:
+        try:
+            # Simple merge
+            y_idx = cols.index('year')
+            m_idx = cols.index('month')
+            df['period'] = df.iloc[:, y_idx].astype(str) + '-' + df.iloc[:, m_idx].astype(str)
+        except: pass
+        
+    return df
+
+# --- 5. PROCESSING CORE ---
+
+def process_file_v6(file_path: Path, engine, client, model):
+    # 1. Skip Check
+    if any(p in file_path.name.lower() for p in SKIP_PATTERNS):
+        logger.warning(f" Skipping {file_path.name} (Blacklisted)")
+        return
+
+    logger.info(f"Processing {file_path.name}...")
+    
+    # 2. Read
+    df_raw = read_csv_robust(file_path)
+    if df_raw.empty: return
+
+    # 3. Identify Expectations for Header Scanning
+    expected_keys = []
+    for key, cols in SCHEMA_DEFINITIONS.items():
+        if key in file_path.name.lower():
+            expected_keys = cols
+            break
+
+    # 4. Find Header
+    header_idx, data_start = find_best_header_row(df_raw, expected_keys)
+    
+    # 5. Extract Headers
+    raw_headers = df_raw.iloc[header_idx].tolist()
+    
+    # Double Header Check
+    if header_idx > 0:
+        row_above = df_raw.iloc[header_idx-1].fillna("").astype(str).tolist()
+        if sum(len(x) for x in row_above) > 10:
+            raw_headers = [f"{p} {c}".strip() for p, c in zip(row_above, raw_headers)]
+
+    if len(raw_headers) != len(df_raw.columns):
+        raw_headers = [f"col_{i}" for i in range(len(df_raw.columns))]
+
+    # 6. LLM / Hybrid Map
+    first_row = df_raw.iloc[data_start].tolist() if data_start < len(df_raw) else [""]*len(raw_headers)
+    clean_headers = get_clean_headers(raw_headers, first_row, file_path.name, client, model)
+    
+    # Align Lengths
+    if len(clean_headers) < len(df_raw.columns):
+        clean_headers += [f"extra_{i}" for i in range(len(df_raw.columns) - len(clean_headers))]
+    clean_headers = clean_headers[:len(df_raw.columns)]
+
+    # 7. Build DF
+    df = df_raw.iloc[data_start:].copy()
+    df.columns = clean_headers
+    
+    # 8. Transforms
+    df = apply_specific_transforms(df, file_path.name)
+    
+    # 9. Clean & Save
+    df = df.loc[:, ~df.columns.str.contains('^unnamed', case=False)]
+    df.dropna(thresh=1, inplace=True)
+    
+    for c in df.columns:
+        if any(x in str(c).lower() for x in ['rate', 'value', 'amount', 'mean', 'buy', 'sell']):
+            df[c] = df[c].apply(lambda x: pd.to_numeric(str(x).replace(',', '').replace('(', '-').replace(')', ''), errors='ignore'))
+
+    table_name = re.sub(r'cbk_batch_\d+_\d+_', '', file_path.stem)
+    table_name = re.sub(r'[^a-zA-Z0-9]', '_', table_name).lower()[:60].lstrip('_')
+    df['source_file'] = file_path.name
+    
+    try:
+        df.to_sql(table_name, engine, if_exists='replace', index=False)
+        logger.info(f"    Saved {len(df)} rows to '{table_name}'")
+    except Exception as e:
+        logger.error(f"    SQL Error: {e}")
+
+# --- MAIN ---
+
+def process_cbk_files(input_dir: str, db_name="mshauri_fedha_v6.db", model="qwen2.5:14b"):
+    SCRATCH = os.environ.get("SCRATCH", "/tmp")
+    BASE_DIR = Path(SCRATCH)
+    INPUT_PATH = Path(input_dir) if Path(input_dir).exists() else BASE_DIR / input_dir
+    
+    if not INPUT_PATH.exists(): return
+
+    OLLAMA_BIN = BASE_DIR / "ollama_core/bin/ollama"
+    CUSTOM_PORT = "25000"
+    OLLAMA_HOST = f"http://127.0.0.1:{CUSTOM_PORT}"
+    
+    if not _manage_ollama_server(OLLAMA_HOST, CUSTOM_PORT, OLLAMA_BIN, model): return
+    
+    engine = create_engine(f"sqlite:///{db_name}")
+    client = Client(host=OLLAMA_HOST)
+    
+    files = sorted(list(INPUT_PATH.glob("*.csv")))
+    print(f"🚀 Processing {len(files)} files...")
+    
+    for f in files:
+        process_file_v6(f, engine, client, model)
+        
+    print("\n Done.")
+    with engine.connect() as conn:
+        tables = conn.execute(text("SELECT name FROM sqlite_master WHERE type='table'")).fetchall()
+        print(f"📊 Created {len(tables)} tables.")
+
+if __name__ == "__main__":
+    pass