fix: Restore exact working structure from commit c72a240

Major changes to match the working HF Spaces deployment:

1. **analyzer/ directory** - Copied src/analyzer/ to analyzer/
- Replaced symlink with actual directory (HF doesn't support symlinks)
- This matches the structure from c72a240 that was working

2. **requirements.txt** - Restored minimal HF-optimized version
- Removed all unnecessary dependencies (FastAPI, MongoDB, PDF, testing, etc.)
- Gradio provided by HF Spaces, not in requirements
- Much faster build times on HF

3. **README.md** - Updated sdk_version to 5.49.1 (from 5.49.0)

This exactly replicates the working deployment from c72a240 before
all the merge/force-push issues started.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

README.md

@@ -4,7 +4,7 @@ emoji: 🎯
 colorFrom: blue
 colorTo: green
 sdk: gradio
-sdk_version: 5.49.0
+sdk_version: 5.49.1
 app_file: app.py
 pinned: false
 license: mit

analyzer

@@ -1 +0,0 @@
-src/analyzer

analyzer/cache/memo.py

@@ -0,0 +1,36 @@
+"""
+cache/memo.py — tiny in-memory and file-backed cache
+"""
+
+from __future__ import annotations
+from pathlib import Path
+from typing import Any, Optional
+import json
+import os
+
+_CACHE: dict[str, Any] = {}
+_CACHE_DIR = Path(".cache")
+_CACHE_DIR.mkdir(exist_ok=True)
+
+def cache_get(key: str) -> Optional[Any]:
+    if key in _CACHE:
+        return _CACHE[key]
+    f = _CACHE_DIR / (key.replace(":", "_") + ".json")
+    if f.exists():
+        try:
+            with open(f, "r", encoding="utf-8") as fh:
+                val = json.load(fh)
+            _CACHE[key] = val
+            return val
+        except Exception:
+            return None
+    return None
+
+def cache_put(key: str, value: Any) -> None:
+    _CACHE[key] = value
+    f = _CACHE_DIR / (key.replace(":", "_") + ".json")
+    try:
+        with open(f, "w", encoding="utf-8") as fh:
+            json.dump(value, fh, ensure_ascii=False, indent=2)
+    except Exception:
+        pass

analyzer/chat/chat_tools.py

@@ -0,0 +1,881 @@
+# src/analyzer/chat/chat_tools.py
+from __future__ import annotations
+import asyncio
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..config import load_config
+from ..llm_client import LLMClient
+from ..prompt_templates import build_prompt
+from ..context_builder import build_context_with_supporting
+from ..utils.errors import DataLoadError, ValidationError, LLMError
+from ..utils.text import clean, to_number
+from ..utils.dates import parse_date, format_date
+from ..summarizer_optimized import (  # NEW: Optimized caching + batch processing
+    SummaryCache,
+    summarize_grants_async,
+    extract_minimal_context,
+)
+
+# Optional: try to load hybrid search index (requires scikit-learn)
+try:
+    from ..search.hybrid_index import load_index, search_by_grant_id
+    HAS_SEARCH_INDEX = True
+except ImportError:
+    HAS_SEARCH_INDEX = False
+    load_index = None
+    search_by_grant_id = None
+
+
+# ---------------------------------------------------------------------
+# Utility helpers (moved to utils modules)
+# ---------------------------------------------------------------------
+# Date parsing: use utils.dates.parse_date() and format_date()
+# Text normalization: use utils.text.clean()
+# Number parsing: use utils.text.to_number()
+
+# Keep backward compatibility wrappers
+_parse_date = parse_date
+_fmt_date = format_date
+
+
+# Helper to normalize complex objects to searchable text
+def _norm(s: Any) -> str:
+    """Normalize any object to searchable text string."""
+    if s is None:
+        return ""
+    if isinstance(s, (list, tuple, set)):
+        return " ".join(_norm(x) for x in s if x)
+    if isinstance(s, dict):
+        return " ".join(_norm(v) for v in s.values() if v)
+    return clean(str(s))  # Use utils.text.clean() for final normalization
+
+
+# ---------------------------------------------------------------------
+# Main ChatTools class
+# ---------------------------------------------------------------------
+@dataclass
+class ChatTools:
+    current: List[Dict[str, Any]]
+    past: List[Dict[str, Any]]
+
+    def __init__(self, current: List[Dict[str, Any]], past: List[Dict[str, Any]]) -> None:
+        self.current = current or []
+        self.past = past or []
+        self.cfg = load_config()
+        try:
+            self.client = LLMClient(self.cfg)
+        except Exception as e:
+            logging.warning("LLMClient init failed: %s", e)
+            self.client = None
+
+        # Try to load hybrid index for enrichment (optional - requires scikit-learn)
+        self.support_idx = None
+        if HAS_SEARCH_INDEX:
+            idx_path = Path("data/index/hybrid_index.pkl")
+            try:
+                self.support_idx = load_index() if idx_path.exists() else None
+            except Exception as e:
+                logging.warning("Could not load search index: %s", e)
+                self.support_idx = None
+
+        # Initialize cache for summaries (NEW: Optimized caching)
+        self.summary_cache = SummaryCache(ttl_seconds=3600)
+
+    # -----------------------------------------------------------------
+    # Cache stats logging helper
+    # -----------------------------------------------------------------
+    def _log_cache_stats(self) -> None:
+        """Log cache statistics for monitoring."""
+        stats = self.summary_cache.stats()
+        logging.info(
+            f"📊 Cache stats: {stats['valid']}/{stats['cached']} valid entries "
+            f"({stats['valid']/max(stats['cached'], 1)*100:.1f}% hit rate)"
+        )
+
+    # -----------------------------------------------------------------
+    # Status calculation (NEW)
+    # -----------------------------------------------------------------
+    def _calculate_grant_status(self, grant: Dict[str, Any]) -> str:
+        """
+        Calculate grant status based on open_date and close_date.
+
+        Returns: "upcoming", "open", or "closed"
+        """
+        today = datetime.now()
+
+        close_date = _parse_date(grant.get("close_date") or grant.get("deadline"))
+        open_date = _parse_date(grant.get("open_date"))
+
+        # If we can't parse dates, assume open
+        if not close_date:
+            return "unknown"
+
+        # If deadline has passed, it's closed
+        if close_date < today:
+            return "closed"
+
+        # If hasn't opened yet, it's upcoming
+        if open_date and open_date > today:
+            return "upcoming"
+
+        # Otherwise it's open
+        return "open"
+
+    # -----------------------------------------------------------------
+    # Listing grants
+    # -----------------------------------------------------------------
+    def list_grants(
+        self,
+        keyword: Optional[str] = None,
+        max_award: Optional[float] = None,
+        audience: Optional[str] = None,
+        status: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        List all grants (or filtered subset) sorted by deadline.
+
+        Args:
+            keyword: Filter by keyword in title/description
+            max_award: Filter by maximum funding ceiling
+            audience: Filter by audience type (not yet implemented)
+            status: Filter by status - "open", "closed", "upcoming", or None for all
+            limit: Max results to return. If None, returns ALL matching grants.
+
+        Returns:
+            List of grant dicts sorted by deadline, with status field included
+        """
+        kw = (keyword or "").lower()
+        results = []
+        for r in self.current:
+            txt = _norm(r)
+            if kw and kw not in txt.lower():
+                continue
+            if max_award is not None:
+                ma = to_number(r.get("max_award") or r.get("funding_max"))
+                if ma and ma > max_award:
+                    continue
+
+            # Calculate status based on dates
+            grant_status = self._calculate_grant_status(r)
+
+            # Filter by status if specified
+            if status is not None and grant_status != status:
+                continue
+
+            results.append(
+                {
+                    "id": r.get("id") or r.get("competition_id"),
+                    "title": r.get("title") or "(untitled)",
+                    "deadline": r.get("deadline") or r.get("close_date") or "n/a",
+                    "status": grant_status,  # NEW: Include status field
+                }
+            )
+        results.sort(key=lambda x: _parse_date(x.get("deadline")) or datetime.max)
+
+        # KEY FIX: Return ALL results if limit is None, not hardcoded 5
+        if limit is None:
+            return results
+        return results[:limit]
+
+    # -----------------------------------------------------------------
+    # Retrieve a single grant
+    # -----------------------------------------------------------------
+    def get_grant(self, gid: str) -> Dict[str, Any]:
+        gid = str(gid).replace("competition-", "").replace("grant-", "").strip().lower()
+        for coll in (self.current, self.past):
+            for r in coll:
+                rid = str(r.get("id") or r.get("competition_id") or "").lower()
+                if rid.replace("competition-", "") == gid:
+                    return r
+        raise KeyError(f"Grant not found: {gid}")
+
+    # -----------------------------------------------------------------
+    # Batch Summarize Multiple Grants (NEW - Parallelized)
+    # -----------------------------------------------------------------
+    async def summarize_grants_batch(
+        self,
+        grant_ids: List[str],
+        include_supporting: bool = False,
+        batch_size: int = 5,
+    ):
+        """
+        Batch summarize multiple grants efficiently using parallel processing.
+
+        This method:
+        1. Resolves grant IDs to grant objects
+        2. Processes them in parallel batches (5 per batch by default)
+        3. Caches results for future use
+        4. Yields results as they complete (parallelized)
+
+        Args:
+            grant_ids: List of grant IDs to summarize
+            include_supporting: If True, include supporting materials (slower)
+            batch_size: Number of grants per batch (default 5)
+
+        Yields:
+            Dict with grant_id, title, summary_md as each completes
+        """
+        import asyncio
+
+        # Resolve all grant IDs to actual grant objects
+        grants_to_summarize = []
+        for gid in grant_ids:
+            try:
+                grant = self.get_grant(gid)
+                grants_to_summarize.append(grant)
+            except KeyError:
+                logging.warning(f"Grant not found: {gid}")
+                continue
+
+        if not grants_to_summarize:
+            logging.warning("No valid grants found to summarize")
+            return
+
+        logging.info(
+            f"📦 Starting batch summarization of {len(grants_to_summarize)} grants "
+            f"(batch_size={batch_size})"
+        )
+
+        # Use the optimized async batch processing function
+        try:
+            results = await summarize_grants_async(
+                grants_to_summarize,
+                past_winners=self.past,
+                client=self.client,
+                cache=self.summary_cache,
+                batch_size=batch_size,
+            )
+            # Yield each result as it's ready
+            for result in results:
+                yield result
+
+            # Log cache stats after batch completion
+            self._log_cache_stats()
+        except Exception as e:
+            logging.error(f"Batch summarization failed: {e}")
+            # Log cache stats even on error
+            self._log_cache_stats()
+            raise
+
+    async def get_all_grant_summaries(self, batch_size: int = 5):
+        """
+        Get summaries of ALL grants in a single efficient batch operation.
+
+        This method:
+        1. Extracts all grant IDs from current database
+        2. Summarizes them all in parallel batches
+        3. Returns all results formatted for display
+
+        Args:
+            batch_size: Number of grants per batch (default 5)
+
+        Yields:
+            Dict with grant_id, title, summary_md as each completes
+        """
+        # Get all grant IDs
+        all_grants = self.list_grants(limit=None)  # Get ALL grants
+        all_grant_ids = [g["id"] for g in all_grants]
+
+        if not all_grant_ids:
+            logging.warning("No grants found in database")
+            return
+
+        logging.info(f"📦 Getting summaries for ALL {len(all_grant_ids)} grants in batch")
+
+        # Use batch summarization with all IDs
+        async for result in self.summarize_grants_batch(all_grant_ids, batch_size=batch_size):
+            yield result
+
+        # Log final cache stats
+        self._log_cache_stats()
+
+    # -----------------------------------------------------------------
+    # Summarize a grant
+    # -----------------------------------------------------------------
+    def summarize_grant(self, gid: str, include_supporting: bool = True, summary_type: str = "layman") -> Dict[str, Any]:
+        """
+        Summarize a grant using LLM with MongoDB and memory caching.
+
+        Args:
+            gid: Grant ID
+            include_supporting: If True, include supporting PDFs and materials in context
+            summary_type: Type of summary to retrieve ("layman", "technical", "exec")
+
+        Returns:
+            Dict with summary_md, title, id
+        """
+        row = self.get_grant(gid)
+        title = row.get("title", "(untitled)")
+        grant_id = row.get("id") or gid
+
+        # NEW: Check MongoDB first for pre-computed summaries
+        try:
+            from ...database import SummaryStore
+            summary_store = SummaryStore()
+            mongodb_summary = summary_store.get_summary(grant_id, summary_type)
+
+            if mongodb_summary:
+                logging.info(f"📦 MongoDB HIT for grant {grant_id} ({summary_type})")
+                return {
+                    "summary_md": mongodb_summary,
+                    "title": title,
+                    "id": grant_id,
+                }
+        except Exception as e:
+            logging.warning(f"MongoDB lookup failed for {grant_id}: {e}")
+            # Continue to memory cache/LLM fallback
+
+        # Check memory cache
+        cached_summary = self.summary_cache.get(row)
+        if cached_summary:
+            logging.info("📦 Memory cache HIT for grant %s", grant_id)
+            return {
+                "summary_md": cached_summary,
+                "title": title,
+                "id": grant_id,
+            }
+
+        # Use enhanced context builder that includes supporting materials
+        if include_supporting:
+            try:
+                context = build_context_with_supporting(row, k=5)
+            except Exception as e:
+                logging.warning("Failed to build context with supporting materials: %s", e)
+                # Fallback to basic context
+                context = self._build_basic_context(row)
+        else:
+            context = self._build_basic_context(row)
+
+        if not self.client or not self.client.is_ready():
+            result = {
+                "summary_md": f"LLM unavailable — context excerpt:\n\n{context[:1000]}",
+                "title": title,
+                "id": grant_id,
+            }
+            self.summary_cache.set(row, result["summary_md"])
+            return result
+
+        payload = build_prompt("openai", context)
+        try:
+            text = self.client.chat(payload["messages"], max_tokens=1200, temperature=0.25)
+            logging.info("✅ Generated summary for grant %s", grant_id)
+        except Exception as e:
+            text = f"LLM error: {e}\n\n{context[:800]}"
+            logging.error("❌ Failed to summarize %s: %s", grant_id, e)
+
+        # NEW: Cache the summary
+        self.summary_cache.set(row, text)
+
+        # Log cache stats
+        self._log_cache_stats()
+
+        return {"summary_md": text, "title": title, "id": grant_id}
+
+    # -----------------------------------------------------------------
+    # Helper method for basic context (without supporting materials)
+    # -----------------------------------------------------------------
+    def _build_basic_context(self, row: Dict[str, Any]) -> str:
+        """Build basic grant context without supporting materials."""
+        title = row.get("title", "(untitled)")
+        url = row.get("url") or row.get("source_url") or ""
+        parts = [
+            f"TITLE: {title}",
+            f"ID: {row.get('id') or row.get('competition_id')}",
+            f"URL: {url}",
+            f"DEADLINE: {_fmt_date(row.get('deadline') or row.get('close_date'))}",
+        ]
+        for k in (
+            "summary",
+            "overview",
+            "scope",
+            "eligibility",
+            "funding",
+            "dates",
+            "how_to_apply",
+            "supporting_information",
+        ):
+            v = row.get(k)
+            if v:
+                parts.append(f"{k.upper()}:\n{_norm(v)}")
+        return "\n".join(parts)
+
+    # -----------------------------------------------------------------
+    # Batch process multiple grants
+    # -----------------------------------------------------------------
+    def batch_process_grants(
+        self,
+        grant_ids: List[str],
+        operation_type: str = "summarize",
+        batch_size: int = 5
+    ) -> Dict[str, str]:
+        """
+        Batch process multiple grants with a single LLM call per batch.
+
+        This method groups grants into batches and sends them as a single prompt
+        with numbered sections, then parses the response to extract individual results.
+
+        Args:
+            grant_ids: List of grant IDs to process
+            operation_type: Type of operation - "summarize", "translate", "simplify", etc.
+            batch_size: Number of grants per batch (default 5)
+
+        Returns:
+            Dictionary mapping grant_id to result text
+
+        Example:
+            results = tools.batch_process_grants(
+                ["competition-2315", "competition-2316"],
+                operation_type="summarize"
+            )
+            # Returns: {"competition-2315": "summary text...", "competition-2316": "..."}
+        """
+        import hashlib
+
+        if not grant_ids:
+            return {}
+
+        if not self.client or not self.client.is_ready():
+            logging.warning("LLM client not available for batch processing")
+            return {gid: "LLM unavailable" for gid in grant_ids}
+
+        # Build operation-specific instruction
+        operation_instructions = {
+            "summarize": "Provide a concise summary highlighting key information, deadlines, and funding details.",
+            "translate": "Translate the grant information into simple, everyday language that anyone can understand.",
+            "simplify": "Explain this grant in layman's terms, avoiding technical jargon.",
+            "analyze": "Analyze this grant's strengths, requirements, and suitability for different applicants.",
+        }
+        instruction = operation_instructions.get(operation_type, "Process this grant information.")
+
+        results = {}
+
+        # Process grants in batches
+        for batch_start in range(0, len(grant_ids), batch_size):
+            batch_ids = grant_ids[batch_start:batch_start + batch_size]
+
+            # Build batch prompt with numbered sections
+            prompt_parts = [
+                f"Process the following {len(batch_ids)} grants. {instruction}",
+                "\nFor each grant, start your response with '### Grant N:' where N is the grant number.",
+                "\n---\n"
+            ]
+
+            # Add each grant with its context
+            grant_contexts = []
+            for idx, grant_id in enumerate(batch_ids, 1):
+                try:
+                    grant = self.get_grant(grant_id)
+                    # Use extract_minimal_context for efficiency
+                    from ..summarizer_optimized import extract_minimal_context
+                    context = extract_minimal_context(grant)
+
+                    prompt_parts.append(f"### Grant {idx}:")
+                    prompt_parts.append(f"ID: {grant_id}")
+                    prompt_parts.append(context)
+                    prompt_parts.append("\n---\n")
+
+                    grant_contexts.append((idx, grant_id))
+
+                except Exception as e:
+                    logging.error(f"Failed to load grant {grant_id}: {e}")
+                    results[grant_id] = f"Error loading grant: {e}"
+
+            if not grant_contexts:
+                continue
+
+            # Build final prompt
+            full_prompt = "\n".join(prompt_parts)
+
+            # Check cache for batch
+            cache_key = hashlib.md5(full_prompt.encode()).hexdigest()[:12]
+            cache_dict = {"id": f"batch_{operation_type}_{cache_key}"}
+            cached_response = self.summary_cache.get(cache_dict)
+
+            if cached_response:
+                logging.info(f"📦 Cache HIT for batch {cache_key}")
+                response_text = cached_response
+            else:
+                # Call LLM with batch prompt
+                try:
+                    messages = [
+                        {"role": "system", "content": "You are a grant analyst. Process each grant separately and clearly mark each response with the grant number."},
+                        {"role": "user", "content": full_prompt}
+                    ]
+                    response_text = self.client.chat(
+                        messages,
+                        max_tokens=batch_size * 400,  # ~400 tokens per grant
+                        temperature=0.3
+                    )
+
+                    # Cache the response
+                    self.summary_cache.set(cache_dict, response_text)
+                    logging.info(f"✅ Batch processed {len(batch_ids)} grants")
+
+                except Exception as e:
+                    logging.error(f"Batch processing failed: {e}")
+                    for _, grant_id in grant_contexts:
+                        results[grant_id] = f"Batch processing error: {e}"
+                    continue
+
+            # Parse response to extract individual results
+            # Use regex to split by "### Grant N:" markers
+            import re
+            pattern = r'### Grant (\d+):(.*?)(?=### Grant \d+:|$)'
+            matches = re.findall(pattern, response_text, re.DOTALL)
+
+            # Map results back to grant IDs
+            for grant_num, result_text in matches:
+                grant_idx = int(grant_num)
+                # Find corresponding grant_id
+                for idx, grant_id in grant_contexts:
+                    if idx == grant_idx:
+                        results[grant_id] = result_text.strip()
+                        break
+
+            # Handle any grants that didn't get matched
+            for idx, grant_id in grant_contexts:
+                if grant_id not in results:
+                    logging.warning(f"No result found for grant {grant_id} (index {idx})")
+                    results[grant_id] = "No response generated"
+
+        # Log cache stats
+        self._log_cache_stats()
+
+        return results
+
+    # -----------------------------------------------------------------
+    # Compare two grants (deterministic)
+    # -----------------------------------------------------------------
+    def compare_grants(self, grant_id_a: str, grant_id_b: str) -> dict:
+        """
+        Deterministic comparison:
+        - Loads both grant records and any supporting index data
+        - Builds a factual side-by-side table from structured fields
+        - Optionally adds a short insight section from the LLM
+        """
+        # ---------------- enrich ----------------
+        def enrich(gid: str) -> dict:
+            base = self.get_grant(gid)
+            row = dict(base)
+            if self.support_idx:
+                hits = search_by_grant_id(
+                    self.support_idx,
+                    gid.replace("competition-", "").replace("grant-", ""),
+                    k=5
+                )
+                # Adapt to new format: [(doc_dict, score), ...]
+                if hits:
+                    doc, score = hits[0]
+                    meta = doc.get("meta", {})
+                    for k, v in meta.items():
+                        if v and k not in row:
+                            row[k] = v
+                    row["_support_text"] = doc.get("text", "")
+            return row
+
+        A = enrich(grant_id_a)
+        B = enrich(grant_id_b)
+
+        def getf(d: dict, key: str) -> str:
+            v = d.get(key) or d.get(key.replace("_", " ")) or ""
+            return str(v).strip() if v not in (None, "", "n/a") else "—"
+
+        # ---------------- table fields ----------------
+        fields = [
+            ("Title", getf(A, "title"), getf(B, "title")),
+            ("Open date", getf(A, "open_date"), getf(B, "open_date")),
+            ("Close date", getf(A, "close_date"), getf(B, "close_date")),
+            (
+                "Funding per project",
+                f"£{getf(A, 'funding_min')}–£{getf(A, 'funding_max')}",
+                f"£{getf(B, 'funding_min')}–£{getf(B, 'funding_max')}",
+            ),
+            ("Total pot", f"£{getf(A, 'total_pot')}", f"£{getf(B, 'total_pot')}"),
+            (
+                "Duration (months)",
+                f"{getf(A, 'duration_min')}–{getf(A, 'duration_max')}",
+                f"{getf(B, 'duration_min')}–{getf(B, 'duration_max')}",
+            ),
+        ]
+
+        table = ["### Side-by-side", "| Field | A | B |", "|---|---|---|"]
+        for name, va, vb in fields:
+            table.append(f"| {name} | {va} | {vb} |")
+
+        # ---------------- optional insight ----------------
+        context_text = ""
+        if "_support_text" in A:
+            context_text += "\n\n[Grant A Supporting Text]\n" + A["_support_text"][:1500]
+        if "_support_text" in B:
+            context_text += "\n\n[Grant B Supporting Text]\n" + B["_support_text"][:1500]
+
+        insight = ""
+        if self.client and self.client.is_ready() and context_text.strip():
+            # Create a cache key for comparison (use sorted grant IDs to ensure consistency)
+            cache_key = {"id": f"compare_{min(grant_id_a, grant_id_b)}_{max(grant_id_a, grant_id_b)}"}
+
+            # Check cache first
+            cached_insight = self.summary_cache.get(cache_key)
+            if cached_insight:
+                logging.info("📦 Cache HIT for comparison %s vs %s", grant_id_a, grant_id_b)
+                insight = cached_insight
+            else:
+                try:
+                    prompt = (
+                        "Given the factual table and context below, write 3-5 bullet points "
+                        "highlighting *meaningful differences* that matter to SMEs (funding size, "
+                        "duration, eligibility, etc.). Do not restate identical facts.\n\n"
+                        + "\n".join(table)
+                        + "\n\n"
+                        + context_text
+                    )
+                    insight = self.client.summarize(prompt)
+                    # Cache the insight
+                    self.summary_cache.set(cache_key, insight)
+                    logging.info("✅ Generated and cached comparison insight")
+                except Exception as e:
+                    logging.warning("compare_grants insight failed: %s", e)
+
+        md = [
+            "### Comparison",
+            f"**{getf(A, 'title')}** _(A)_ vs **{getf(B, 'title')}** _(B)_",
+            "\n".join(table),
+        ]
+        if insight:
+            md += ["\n### Key differences", insight]
+
+        # Log cache stats
+        self._log_cache_stats()
+
+        return {"comparison_md": "\n".join(md)}
+
+    # -----------------------------------------------------------------
+    # Deadlines overview
+    # -----------------------------------------------------------------
+    def deadlines_overview(self, n: Optional[int] = None) -> List[Dict[str, Any]]:
+        """
+        Get upcoming grant deadlines sorted by date.
+
+        If n is None, returns all deadlines. Otherwise returns top n.
+        """
+        rows = []
+        for r in self.current:
+            d = _parse_date(r.get("deadline") or r.get("close_date"))
+            if not d:
+                continue
+            status = self._calculate_grant_status(r)
+            rows.append(
+                {
+                    "id": r.get("id") or r.get("competition_id"),
+                    "title": r.get("title") or "(untitled)",
+                    "deadline": d.strftime("%Y-%m-%d %H:%M:%S"),
+                    "status": status,  # NEW: Include status
+                }
+            )
+        rows.sort(key=lambda x: _parse_date(x["deadline"]) or datetime.max)
+
+        # Default to 5 if not specified (for backward compat with UI)
+        if n is None:
+            n = 5
+        return rows[:n]
+
+    # -----------------------------------------------------------------
+    # Analyze company for grant matching
+    # -----------------------------------------------------------------
+    def analyze_company_for_grants(self, company_url: str, limit: int = 3) -> Dict[str, Any]:
+        """
+        Fetch a company website and analyze which grants would be most suitable.
+
+        Args:
+            company_url: URL of the company website to analyze
+            limit: Number of grant recommendations to return
+
+        Returns:
+            Dict with company analysis and recommended grants
+        """
+        import urllib.request
+        from html.parser import HTMLParser
+
+        # Improved HTML to text parser that handles scripts/styles
+        class HTMLTextExtractor(HTMLParser):
+            def __init__(self):
+                super().__init__()
+                self.text = []
+                self.skip_tags = set()
+
+            def handle_starttag(self, tag, attrs):
+                # Skip script, style, noscript tags
+                if tag in ('script', 'style', 'noscript', 'svg'):
+                    self.skip_tags.add(tag)
+
+            def handle_endtag(self, tag):
+                self.skip_tags.discard(tag)
+
+            def handle_data(self, data):
+                # Only add text if not in skip tags
+                if not self.skip_tags:
+                    stripped = data.strip()
+                    if stripped and len(stripped) > 3:  # Filter out single chars
+                        self.text.append(stripped)
+
+            def get_text(self):
+                return ' '.join(self.text)
+
+        try:
+            # Fetch the website
+            logging.info(f"Fetching company website: {company_url}")
+
+            # Add scheme if missing
+            if not company_url.startswith(('http://', 'https://')):
+                company_url = 'https://' + company_url
+
+            # Set a realistic user agent to avoid blocks
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+            }
+            req = urllib.request.Request(company_url, headers=headers)
+
+            # Fetch with timeout
+            with urllib.request.urlopen(req, timeout=15) as response:
+                html = response.read().decode('utf-8', errors='ignore')
+
+            # Extract text from HTML
+            parser = HTMLTextExtractor()
+            parser.feed(html)
+            company_text = parser.get_text()
+
+            # Clean and truncate (keep first 4000 chars for analysis)
+            company_text = ' '.join(company_text.split())[:4000]
+
+            logging.info(f"Extracted {len(company_text)} chars from {company_url}")
+
+            # Debug: log first 200 chars
+            logging.debug(f"First 200 chars: {company_text[:200]}")
+
+        except Exception as e:
+            logging.error(f"Failed to fetch company website: {e}")
+            return {
+                "error": f"Could not fetch website: {str(e)}",
+                "company_url": company_url,
+                "recommendations": []
+            }
+
+        # Use LLM to analyze company and match with grants
+        if not self.client or not self.client.is_ready():
+            return {
+                "error": "LLM not available for analysis",
+                "company_url": company_url,
+                "recommendations": []
+            }
+
+        try:
+            # Get list of available grants
+            available_grants = []
+            for r in self.current[:20]:  # Limit to 20 grants for context
+                available_grants.append({
+                    "id": r.get("id") or r.get("competition_id"),
+                    "title": r.get("title", "(untitled)"),
+                    "summary": r.get("summary", "")[:200],
+                    "scope": r.get("scope", "")[:200],
+                    "funding_max": r.get("funding_max") or r.get("max_award"),
+                    "deadline": r.get("deadline") or r.get("close_date")
+                })
+
+            # Build analysis prompt
+            grants_context = "\n".join([
+                f"- {g['id']}: {g['title']} (max funding: £{g['funding_max']}, deadline: {g['deadline']})"
+                for g in available_grants
+            ])
+
+            prompt = f"""Analyze this company website and recommend the most suitable grants.
+
+COMPANY WEBSITE TEXT:
+{company_text}
+
+AVAILABLE GRANTS:
+{grants_context}
+
+Based on the company's activities, industry, and apparent needs, which grants would be most suitable?
+Provide your answer in this format:
+
+COMPANY ANALYSIS:
+[Brief 2-3 sentence analysis of what the company does]
+
+RECOMMENDED GRANTS:
+1. [Grant ID]: [Grant Title]
+   - Why: [1-2 sentence explanation of fit]
+
+2. [Grant ID]: [Grant Title]
+   - Why: [1-2 sentence explanation of fit]
+
+3. [Grant ID]: [Grant Title]
+   - Why: [1-2 sentence explanation of fit]
+"""
+
+            # Create cache key for company analysis (hash the URL)
+            import hashlib
+            url_hash = hashlib.md5(company_url.encode()).hexdigest()[:12]
+            cache_key = {"id": f"company_{url_hash}"}
+
+            # Check cache first
+            cached_analysis = self.summary_cache.get(cache_key)
+            if cached_analysis:
+                logging.info("📦 Cache HIT for company analysis: %s", company_url)
+                analysis_text = cached_analysis
+            else:
+                # Get LLM analysis
+                analysis_text = self.client.summarize(prompt)
+                # Cache the analysis
+                self.summary_cache.set(cache_key, analysis_text)
+                logging.info("✅ Generated and cached company analysis")
+
+            # Extract recommended grant IDs from the response
+            recommended_ids = []
+            for line in analysis_text.split('\n'):
+                # Look for patterns like "1. competition-2313:" or "- 2313:"
+                match = re.search(r'(?:competition-)?(\d{4})', line)
+                if match:
+                    gid = match.group(1)
+                    if gid not in recommended_ids:
+                        recommended_ids.append(gid)
+                        if len(recommended_ids) >= limit:
+                            break
+
+            # Get full details for recommended grants
+            recommendations = []
+            for gid in recommended_ids:
+                try:
+                    grant = self.get_grant(gid)
+                    recommendations.append({
+                        "id": grant.get("id") or grant.get("competition_id"),
+                        "title": grant.get("title"),
+                        "deadline": grant.get("deadline") or grant.get("close_date"),
+                        "funding_max": grant.get("funding_max") or grant.get("max_award"),
+                    })
+                except KeyError:
+                    continue
+
+            # Log cache stats
+            self._log_cache_stats()
+
+            return {
+                "company_url": company_url,
+                "analysis": analysis_text,
+                "recommendations": recommendations
+            }
+
+        except Exception as e:
+            logging.error(f"Failed to analyze company: {e}")
+            # Log cache stats even on error
+            self._log_cache_stats()
+            return {
+                "error": f"Analysis failed: {str(e)}",
+                "company_url": company_url,
+                "recommendations": []
+            }

analyzer/chat/company_analyzer.py

@@ -0,0 +1,384 @@
+"""
+Enhanced company analysis for grant matching v2.
+
+This module provides intelligent company profiling and grant matching based on:
+- Technology stack and industry sector
+- Company size and stage
+- Location and eligibility requirements
+- Funding amounts and project types
+"""
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Dict, List, Optional, Set, Any
+
+from ..net.fetcher import fetch_link
+from ..utils.text import clean
+from ..utils.dates import parse_date
+
+logger = logging.getLogger(__name__)
+
+
+# Technology keywords for sector detection
+TECH_KEYWORDS = {
+    "ai_ml": ["ai", "artificial intelligence", "machine learning", "ml", "deep learning",
+              "neural network", "llm", "gpt", "nlp", "computer vision", "agentic"],
+    "battery_ev": ["battery", "batteries", "electric vehicle", "ev", "electrification",
+                   "energy storage", "lithium", "zero emission"],
+    "biotech": ["biotech", "pharmaceutical", "drug discovery", "clinical", "medical device",
+                "diagnostic", "therapeutic", "genomic", "bioinformatics"],
+    "manufacturing": ["manufacturing", "production", "factory", "industrial", "assembly",
+                      "automation", "robotics", "supply chain"],
+    "software": ["software", "saas", "platform", "application", "app", "digital", "cloud"],
+    "green_tech": ["sustainability", "renewable", "green energy", "climate", "carbon",
+                   "environmental", "circular economy", "net zero"],
+    "aerospace": ["aerospace", "aviation", "aircraft", "satellite", "space", "drone"],
+    "quantum": ["quantum computing", "quantum", "qubit"],
+}
+
+# Company size indicators
+SIZE_INDICATORS = {
+    "startup": ["startup", "founded in 202", "seed", "pre-seed", "early stage"],
+    "scale_up": ["scale-up", "scaleup", "series a", "series b", "growing", "expansion"],
+    "sme": ["small business", "sme", "small to medium", "limited", "ltd"],
+    "enterprise": ["enterprise", "corporation", "plc", "publicly traded", "fortune"],
+}
+
+# Location keywords for UK eligibility
+UK_LOCATIONS = [
+    "uk", "united kingdom", "london", "manchester", "birmingham", "glasgow", "edinburgh",
+    "bristol", "leeds", "liverpool", "cardiff", "belfast", "scotland", "wales", "england",
+    "northern ireland", "britain", "british"
+]
+
+
+@dataclass
+class CompanyProfile:
+    """Extracted company profile for grant matching."""
+    url: str
+    text_content: str
+
+    # Detected attributes
+    sectors: Set[str]
+    tech_stack: Set[str]
+    company_size: Optional[str]
+    is_uk_based: bool
+    keywords: Set[str]
+
+    # Inferred characteristics
+    appears_r_and_d_focused: bool
+    mentions_funding: bool
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "url": self.url,
+            "sectors": list(self.sectors),
+            "tech_stack": list(self.tech_stack),
+            "company_size": self.company_size,
+            "is_uk_based": self.is_uk_based,
+            "appears_r_and_d_focused": self.appears_r_and_d_focused,
+            "mentions_funding": self.mentions_funding,
+            "keywords": list(self.keywords)[:20],  # Limit for readability
+        }
+
+
+def extract_company_profile(company_url: str) -> Optional[CompanyProfile]:
+    """
+    Extract detailed company profile from website.
+
+    Args:
+        company_url: URL of company website
+
+    Returns:
+        CompanyProfile with extracted attributes, or None if fetch fails
+    """
+    logger.info(f"Extracting company profile from: {company_url}")
+
+    # Use the existing fetcher
+    result = fetch_link(company_url)
+
+    if not result.get("ok"):
+        logger.error(f"Failed to fetch {company_url}: {result.get('error')}")
+        return None
+
+    text = result.get("text", "")
+    text_lower = text.lower()
+
+    # Detect technology sectors
+    sectors = set()
+    tech_stack = set()
+    for sector, keywords in TECH_KEYWORDS.items():
+        for keyword in keywords:
+            if keyword in text_lower:
+                sectors.add(sector)
+                tech_stack.add(keyword)
+
+    # Detect company size
+    company_size = None
+    for size_type, indicators in SIZE_INDICATORS.items():
+        for indicator in indicators:
+            if indicator in text_lower:
+                company_size = size_type
+                break
+        if company_size:
+            break
+
+    # Check UK location
+    is_uk_based = any(loc in text_lower for loc in UK_LOCATIONS)
+
+    # Extract meaningful keywords (simple approach)
+    words = re.findall(r'\b[a-z]{4,}\b', text_lower)
+    # Filter out common words
+    common_words = {"about", "their", "with", "from", "that", "this", "have", "more",
+                    "what", "when", "where", "which", "they", "would", "could", "should"}
+    keywords = set(w for w in words if w not in common_words)
+
+    # Detect R&D focus
+    r_and_d_indicators = ["research", "development", "innovation", "r&d", "patent",
+                          "prototype", "pilot", "feasibility", "experimental"]
+    appears_r_and_d_focused = any(ind in text_lower for ind in r_and_d_indicators)
+
+    # Check if they mention funding
+    funding_indicators = ["funding", "investment", "grant", "raise", "capital", "finance"]
+    mentions_funding = any(ind in text_lower for ind in funding_indicators)
+
+    profile = CompanyProfile(
+        url=company_url,
+        text_content=text[:5000],  # Keep first 5000 chars for analysis
+        sectors=sectors,
+        tech_stack=tech_stack,
+        company_size=company_size,
+        is_uk_based=is_uk_based,
+        keywords=keywords,
+        appears_r_and_d_focused=appears_r_and_d_focused,
+        mentions_funding=mentions_funding,
+    )
+
+    logger.info(f"Profile extracted: {len(sectors)} sectors, size={company_size}, UK={is_uk_based}")
+    return profile
+
+
+@dataclass
+class GrantMatch:
+    """Represents a grant match with scoring and reasoning."""
+    grant_id: str
+    grant_title: str
+    match_score: float  # 0-100
+    match_category: str  # "perfect", "strong", "potential"
+    reasons: List[str]
+    concerns: List[str]
+    deadline: Optional[str]
+    funding_max: Optional[float]
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "grant_id": self.grant_id,
+            "grant_title": self.grant_title,
+            "match_score": round(self.match_score, 1),
+            "match_category": self.match_category,
+            "reasons": self.reasons,
+            "concerns": self.concerns if self.concerns else None,
+            "deadline": self.deadline,
+            "funding_max": self.funding_max,
+        }
+
+
+def calculate_grant_status(grant: Dict[str, Any]) -> str:
+    """Calculate grant status based on dates."""
+    today = datetime.now()
+    close_date = parse_date(grant.get("close_date") or grant.get("deadline"))
+    open_date = parse_date(grant.get("open_date"))
+
+    if not close_date:
+        return "unknown"
+    if close_date < today:
+        return "closed"
+    if open_date and open_date > today:
+        return "upcoming"
+    return "open"
+
+
+def score_grant_match(
+    profile: CompanyProfile,
+    grant: Dict[str, Any]
+) -> GrantMatch:
+    """
+    Score how well a grant matches a company profile.
+
+    Returns:
+        GrantMatch with score, category, reasons, and concerns
+    """
+    grant_id = grant.get("id") or grant.get("competition_id") or "unknown"
+    grant_title = grant.get("title", "(untitled)")
+
+    # Prepare grant text for analysis
+    grant_text = " ".join([
+        str(grant.get("title", "")),
+        str(grant.get("summary", "")),
+        str(grant.get("scope", "")),
+        str(grant.get("eligibility", "")),
+    ]).lower()
+
+    score = 0.0
+    reasons = []
+    concerns = []
+
+    # 1. Sector/Technology alignment (40 points max)
+    sector_matches = []
+    for sector in profile.sectors:
+        sector_keywords = TECH_KEYWORDS.get(sector, [])
+        for keyword in sector_keywords:
+            if keyword in grant_text:
+                sector_matches.append(keyword)
+
+    if sector_matches:
+        sector_score = min(40, len(sector_matches) * 10)
+        score += sector_score
+        reasons.append(f"Strong sector alignment: {', '.join(set(sector_matches[:3]))}")
+
+    # 2. UK eligibility (20 points if UK-based)
+    grant_status = calculate_grant_status(grant)
+
+    if profile.is_uk_based:
+        score += 20
+        reasons.append("UK-based company (eligible for Innovate UK)")
+    else:
+        concerns.append("Company may not be UK-based (verify eligibility)")
+
+    # 3. Grant status (20 points if open)
+    if grant_status == "open":
+        score += 20
+        reasons.append(f"Grant is currently open")
+    elif grant_status == "upcoming":
+        score += 15
+        reasons.append(f"Grant opens soon")
+    elif grant_status == "closed":
+        score -= 30
+        concerns.append("Grant deadline has passed")
+
+    # 4. Company size/stage fit (10 points)
+    eligibility_text = str(grant.get("eligibility", "")).lower()
+    if profile.company_size:
+        size_mentioned = profile.company_size in eligibility_text
+        if "sme" in eligibility_text or "small" in eligibility_text:
+            if profile.company_size in ["startup", "sme", "scale_up"]:
+                score += 10
+                reasons.append(f"Good fit for {profile.company_size}s")
+        elif size_mentioned:
+            score += 10
+            reasons.append(f"Mentions {profile.company_size}s")
+
+    # 5. R&D focus alignment (10 points)
+    if profile.appears_r_and_d_focused:
+        r_and_d_in_grant = any(word in grant_text for word in ["research", "development", "innovation", "r&d"])
+        if r_and_d_in_grant:
+            score += 10
+            reasons.append("R&D-focused opportunity (matches company profile)")
+
+    # 6. Funding amount considerations
+    funding_max = grant.get("funding_max") or grant.get("max_award")
+    if funding_max:
+        try:
+            funding_val = float(str(funding_max).replace(",", "").replace("£", ""))
+            if funding_val > 1000000:  # £1M+
+                if profile.company_size == "startup":
+                    concerns.append(f"Large grant (£{funding_val:,.0f}) - may require significant match funding")
+        except:
+            pass
+
+    # Determine match category
+    if score >= 70:
+        category = "perfect"
+    elif score >= 50:
+        category = "strong"
+    elif score >= 30:
+        category = "potential"
+    else:
+        category = "weak"
+
+    return GrantMatch(
+        grant_id=grant_id,
+        grant_title=grant_title,
+        match_score=score,
+        match_category=category,
+        reasons=reasons,
+        concerns=concerns if concerns else [],
+        deadline=grant.get("deadline") or grant.get("close_date"),
+        funding_max=funding_max,
+    )
+
+
+def analyze_company_for_grants_v2(
+    company_url: str,
+    grants: List[Dict[str, Any]],
+    limit: int = 10
+) -> Dict[str, Any]:
+    """
+    Enhanced company analysis with smart grant matching.
+
+    Args:
+        company_url: URL of company website
+        grants: List of available grants to match against
+        limit: Maximum number of recommendations per category
+
+    Returns:
+        Dict with company profile, perfect matches, strong matches, and exclusion reasons
+    """
+    logger.info(f"Starting enhanced company analysis for: {company_url}")
+
+    # Extract company profile
+    profile = extract_company_profile(company_url)
+    if not profile:
+        return {
+            "error": "Failed to extract company profile from URL",
+            "company_url": company_url,
+        }
+
+    # Score all grants
+    all_matches = []
+    for grant in grants:
+        match = score_grant_match(profile, grant)
+        all_matches.append(match)
+
+    # Sort by score
+    all_matches.sort(key=lambda m: m.match_score, reverse=True)
+
+    # Categorize matches
+    perfect_matches = [m for m in all_matches if m.match_category == "perfect"][:limit]
+    strong_matches = [m for m in all_matches if m.match_category == "strong"][:limit]
+    potential_matches = [m for m in all_matches if m.match_category == "potential"][:limit]
+
+    # Explain why others were excluded (top reasons)
+    excluded = [m for m in all_matches if m.match_category == "weak"]
+    exclusion_reasons = {}
+    for match in excluded[:10]:  # Analyze top 10 excluded
+        for concern in match.concerns:
+            if concern not in exclusion_reasons:
+                exclusion_reasons[concern] = 0
+            exclusion_reasons[concern] += 1
+
+    # Sort exclusion reasons by frequency
+    top_exclusions = sorted(exclusion_reasons.items(), key=lambda x: x[1], reverse=True)[:5]
+
+    return {
+        "company_url": company_url,
+        "company_profile": profile.to_dict(),
+        "perfect_matches": [m.to_dict() for m in perfect_matches],
+        "strong_matches": [m.to_dict() for m in strong_matches],
+        "worth_considering": [m.to_dict() for m in potential_matches],
+        "why_not_others": {
+            "top_exclusion_reasons": [reason for reason, count in top_exclusions],
+            "total_excluded": len(excluded),
+        },
+        "summary": {
+            "total_analyzed": len(all_matches),
+            "perfect_matches_count": len(perfect_matches),
+            "strong_matches_count": len(strong_matches),
+            "potential_matches_count": len(potential_matches),
+        }
+    }

analyzer/chat/demo_app.py

@@ -0,0 +1,860 @@
+#!/usr/bin/env python3
+"""
+demo_app.py — Client-ready demo interface with preset questions
+
+A Gradio web app for testing the grant analyst with clients.
+Includes 5 preset questions that are guaranteed to work.
+
+Usage:
+    python -m src.analyzer.chat.demo_app
+    python -m src.analyzer.chat.demo_app --share  # Create public URL
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+try:
+    import gradio as gr
+except ImportError:
+    print("ERROR: Gradio not installed. Install with: pip install gradio")
+    sys.exit(1)
+
+try:
+    import httpx
+except ImportError:
+    print("ERROR: httpx not installed. Install with: pip install httpx")
+    httpx = None
+
+
+from ..config import load_config
+from ..data_loader import load_current_grants, load_past_winners
+from ..llm_client import LLMClient
+from ..search.hybrid_index import load_index
+from .chat_tools import ChatTools
+from .tool_schemas import openai_tools, detect_extended_features
+from ..utils.query_logger import get_query_logger
+from ..summarizer_optimized import SummaryCache  # NEW: Optimized caching
+import asyncio  # NEW: For batch processing
+
+
+# Preset questions that are guaranteed to work
+PRESET_QUESTIONS = {
+    "List battery grants": "List all grants related to batteries",
+    "Show upcoming deadlines": "What are the upcoming grant deadlines?",
+    "Compare two grants": "Compare competition-2313 and competition-2314",
+    "Grant details": "Tell me about competition-2317 in detail",
+    "SME funding options": "What grants are available for SMEs with funding over £100k?"
+}
+
+
+class GrantAnalystDemo:
+    """Demo application state manager."""
+
+    def __init__(self):
+        """Initialize the demo application."""
+        self.cfg = None
+        self.llm_client = None
+        self.tools = None
+        self.available_tools = []
+        self.messages = []
+        self.initialized = False
+        self.summary_cache = SummaryCache(ttl_seconds=3600)  # NEW: Cache summaries for 1 hour
+
+    def initialize(self) -> Tuple[bool, str]:
+        """
+        Initialize all components.
+
+        Returns:
+            (success, message) tuple
+        """
+        try:
+            # Load config
+            self.cfg = load_config()
+
+            # Load data
+            current = load_current_grants(Path("data/snapshots"))
+
+            # Try to load past winners (optional - graceful fallback)
+            past = []
+            try:
+                xlsx_path = Path("data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx")
+                if xlsx_path.exists():
+                    past = load_past_winners(history_xlsx=xlsx_path)
+                    logging.info(f"Loaded {len(past)} past winners")
+            except Exception as e:
+                logging.warning(f"Could not load past winners: {e}")
+                past = []
+
+            # Initialize LLM
+            self.llm_client = LLMClient(self.cfg)
+            if not self.llm_client.is_ready():
+                return False, "ERROR: LLM client not ready. Check API key configuration."
+
+            # Load index
+            try:
+                idx_path = Path("data/index/hybrid_index.pkl")
+                if idx_path.exists():
+                    _ = load_index()
+            except Exception as e:
+                logging.warning(f"Could not load search index: {e}")
+
+            # Initialize tools
+            self.tools = ChatTools(current, past)
+
+            # Register tools
+            extended_mode = detect_extended_features()
+            self.available_tools = openai_tools(extended=extended_mode)
+
+            # Log which tools are available
+            tool_names = [t["function"]["name"] for t in self.available_tools]
+            logging.info(f"Loaded {len(tool_names)} tools: {', '.join(tool_names)}")
+            if extended_mode:
+                logging.info("Extended tools ENABLED (fetch_link, insight_search)")
+            else:
+                logging.info("Extended tools DISABLED (run with ENABLE_EXTENDED_TOOLS=1 to enable)")
+
+            # Initialize conversation
+            self.messages = [
+                {
+                    "role": "system",
+                    "content": (
+                        "You are an expert UK grant analyst assistant. Your PRIMARY DIRECTIVE is to EXECUTE user requests.\n\n"
+                        "WHEN USER ASKS FOR:\n"
+                        "- 'description/summaries of all/every grant' → IMMEDIATELY call get_all_grant_summaries (ONE SINGLE TOOL CALL)\n"
+                        "- 'description/summaries of grants' → IMMEDIATELY call summarize_grants_batch\n"
+                        "- 'list all grants' (NO descriptions) → IMMEDIATELY call list_grants with limit=None\n"
+                        "- 'find grants about [topic]' → IMMEDIATELY call search_grants\n"
+                        "- ANY REQUEST FOR INFORMATION → DO NOT DESCRIBE WHAT YOU WILL DO, JUST DO IT\n\n"
+                        "CRITICAL RULES:\n"
+                        "- DO NOT make multiple tool calls. Make ONE tool call and wait for results.\n"
+                        "- DO NOT return raw JSON lists when user asks for descriptions/summaries\n"
+                        "- When user asks for 'all grants', use get_all_grant_summaries (NOT list_grants + summarize)\n"
+                        "- DO NOT say 'I will do X' and then stop. ACTUALLY CALL THE TOOL.\n"
+                        "- DO NOT provide preliminary responses. CALL TOOLS FIRST, THEN RESPOND.\n"
+                        "- If user asks for information, ALWAYS use tools - NEVER make up answers.\n"
+                        "- Never promise to do something later. Do it immediately.\n\n"
+                        "SPECIFIC TOOL USAGE:\n"
+                        "- get_all_grant_summaries: For 'all grants', 'every grant', 'all grant opportunities' (ONE SINGLE CALL - most efficient)\n"
+                        "- summarize_grants_batch: For summaries/descriptions of specific grant groups\n"
+                        "- summarize_grant: Only for single grant details\n"
+                        "- list_grants: To get IDs/titles only (NOT for descriptions)\n"
+                        "- search_grants: For finding grants by topic/keyword\n"
+                        "- get_grant: For full structured data on one grant\n"
+                        "- compare_grants: For side-by-side comparisons\n\n"
+                        "RESPONSE FORMAT:\n"
+                        "- ALWAYS include complete tool results in your response\n"
+                        "- Do NOT paraphrase or summarize tool results - display them exactly as provided\n"
+                        "- Use markdown formatting (headers, numbered lists, tables)\n"
+                        "- When displaying lists of grants, ALWAYS use numbered format (1., 2., 3., etc.) NOT bullet points\n"
+                        "- Include all details: funding, eligibility, deadlines, scope\n"
+                        "- No length limits - be comprehensive\n"
+                        "- NEVER omit tool results from your response\n\n"
+                        "Current date: 2025-10-27"
+                    )
+                }
+            ]
+
+            self.initialized = True
+            return True, f"Loaded {len(current)} grants with {len(self.available_tools)} tools"
+
+        except Exception as e:
+            logging.error(f"Initialization failed: {e}", exc_info=True)
+            return False, f"ERROR: Initialization failed: {e}"
+
+    def _dispatch_tool(self, tool_name: str, tool_args: Dict) -> any:
+        """Execute a tool call."""
+        try:
+            if tool_name == "list_grants":
+                return self.tools.list_grants(
+                    keyword=tool_args.get("keyword"),
+                    max_award=tool_args.get("max_award"),
+                    audience=tool_args.get("audience"),
+                    status=tool_args.get("status"),  # NEW: Add status filter
+                    limit=tool_args.get("limit")  # FIXED: Don't default to 5, pass None for all
+                )
+
+            elif tool_name == "get_grant":
+                return self.tools.get_grant(tool_args["grant_id"])
+
+            elif tool_name == "summarize_grant":
+                return self.tools.summarize_grant(tool_args["grant_id"])
+
+            elif tool_name == "summarize_grants_batch":
+                # NEW: Batch summarization with parallel processing
+                grant_ids = tool_args.get("grant_ids", [])
+                batch_size = tool_args.get("batch_size", 5)
+
+                if not grant_ids:
+                    return "ERROR: No grant IDs provided for batch summarization"
+
+                logging.info(f"Starting batch summarization of {len(grant_ids)} grants")
+
+                # Collect results from async generator
+                results = []
+                try:
+                    loop = asyncio.get_event_loop()
+                except RuntimeError:
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+
+                async def collect_batch_results():
+                    """Collect all batch results."""
+                    async for result in self.tools.summarize_grants_batch(
+                        grant_ids,
+                        batch_size=batch_size
+                    ):
+                        results.append(result)
+
+                try:
+                    loop.run_until_complete(collect_batch_results())
+                except RuntimeError as e:
+                    if "already running" in str(e):
+                        # If loop is already running (shouldn't happen in Gradio), use current loop
+                        logging.warning(f"Event loop already running, using current loop")
+                        # In this case, we need to return a message instead
+                        return "WARNING: Batch summarization not available in this context. Please try individual summaries."
+                    raise
+
+                # Format results for display
+                if not results:
+                    return "ERROR: No grants could be summarized"
+
+                formatted = f"Batch summarization complete for {len(results)} grants:\n\n"
+                for i, result in enumerate(results, 1):
+                    title = result.get("title", "(untitled)")
+                    summary = result.get("summary_md", "No summary")
+                    # Truncate long summaries for display
+                    if len(summary) > 500:
+                        summary = summary[:500] + "\n\n[... truncated ...]"
+                    formatted += f"**{i}. {title}**\n{summary}\n\n---\n\n"
+
+                return formatted
+
+            elif tool_name == "get_all_grant_summaries":
+                # Get summaries of ALL grants in one batch
+                batch_size = tool_args.get("batch_size", 5)
+
+                logging.info(f"Starting to get summaries for ALL grants (batch_size={batch_size})")
+
+                # Collect results from async generator
+                results = []
+                try:
+                    loop = asyncio.get_event_loop()
+                except RuntimeError:
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+
+                async def collect_all_summaries():
+                    """Collect all grant summaries."""
+                    async for result in self.tools.get_all_grant_summaries(batch_size=batch_size):
+                        results.append(result)
+
+                try:
+                    loop.run_until_complete(collect_all_summaries())
+                except RuntimeError as e:
+                    if "already running" in str(e):
+                        logging.warning(f"Event loop already running, using current loop")
+                        return "WARNING: Cannot get all summaries in this context. Please try specific summaries."
+                    raise
+
+                # Format results for display
+                if not results:
+                    return "ERROR: No grants could be summarized"
+
+                formatted = f"Summaries for ALL {len(results)} grants:\n\n"
+                for i, result in enumerate(results, 1):
+                    title = result.get("title", "(untitled)")
+                    summary = result.get("summary_md", "No summary")
+                    formatted += f"**{i}. {title}**\n{summary}\n\n---\n\n"
+
+                return formatted
+
+            elif tool_name == "compare_grants":
+                return self.tools.compare_grants(
+                    tool_args["grant_id_a"],
+                    tool_args["grant_id_b"]
+                )
+
+            elif tool_name == "deadlines_overview":
+                return self.tools.deadlines_overview(tool_args.get("n", 5))
+
+            elif tool_name == "analyze_company_for_grants":
+                return self.tools.analyze_company_for_grants(
+                    tool_args["company_url"],
+                    limit=tool_args.get("limit", 3)
+                )
+
+            elif tool_name == "search_grants":
+                results = self.tools.list_grants(
+                    keyword=tool_args.get("query"),
+                    status=tool_args.get("status"),
+                    limit=tool_args.get("limit")  # If None, returns ALL
+                )
+                # Format results for better conversation flow (avoid bloating history)
+                if len(results) > 50:
+                    # If too many, return summary + first 20
+                    summary = f"Found {len(results)} grants matching the criteria. Showing first 20:\n"
+                    display = results[:20]
+                else:
+                    summary = f"Found {len(results)} grants:\n"
+                    display = results
+
+                formatted = summary + "\n".join([
+                    f"  • {r['title'][:60]} (ID: {r['id']}, Deadline: {r['deadline']}, Status: {r['status']})"
+                    for r in display
+                ])
+                return formatted
+
+            elif tool_name == "fetch_link":
+                # NEW: Handle external link fetching
+                try:
+                    from ..net.fetcher import fetch_link
+                    url = tool_args.get("url")
+                    if not url:
+                        return {"error": "No URL provided"}
+
+                    logging.info(f"Fetching external link: {url}")
+                    content = fetch_link(url)
+                    if not content:
+                        return {"error": f"Could not fetch content from {url}"}
+
+                    # Truncate very long content to avoid bloating conversation
+                    if len(content) > 10000:
+                        content = content[:10000] + "\n\n[... content truncated ...]"
+
+                    return content
+                except ImportError:
+                    return {"error": "Link fetching not available"}
+                except Exception as e:
+                    logging.error(f"Failed to fetch link {tool_args.get('url')}: {e}")
+                    return {"error": f"Failed to fetch link: {str(e)[:200]}"}
+
+            else:
+                return {"error": f"Unknown tool: {tool_name}"}
+
+        except Exception as e:
+            logging.error(f"Tool {tool_name} failed: {e}", exc_info=True)
+            return {"error": str(e)}
+
+    def chat_stream(self, user_message: str, history: List, use_sse: bool = False, use_websocket: bool = False):
+        """
+        Process a chat message with optional SSE or WebSocket streaming.
+
+        Args:
+            user_message: User's input
+            history: Gradio chat history
+            use_sse: If True, use SSE streaming from API endpoint
+            use_websocket: If True, use WebSocket streaming (takes precedence over SSE)
+
+        Yields:
+            Updated history for streaming response
+        """
+        import time
+        import json
+
+        if not self.initialized:
+            yield history + [[user_message, "WARNING: System not initialized. Please restart the app."]]
+            return
+
+        # WebSocket streaming takes precedence
+        if use_websocket and HAS_WEBSOCKET:
+            try:
+                accumulated_response = ""
+                intent = None
+                citations = []
+
+                # Connect to WebSocket endpoint
+                ws = websocket.create_connection("ws://localhost:8000/ws/query", timeout=60)
+
+                # Send query
+                message = {
+                    "query": user_message,
+                    "session_id": "gradio_session"
+                }
+                ws.send(json.dumps(message))
+
+                # Receive and process stream
+                while True:
+                    try:
+                        msg = ws.recv()
+                        data = json.loads(msg)
+                        msg_type = data.get("type")
+
+                        if msg_type == "metadata":
+                            # Initial metadata received
+                            logging.info(f"WebSocket session: {data.get('session_id')}")
+
+                        elif msg_type == "intent":
+                            intent = data.get("intent")
+                            # Show intent in response
+                            status_msg = f"*Detected intent: {intent}*\n\n"
+                            yield history + [[user_message, status_msg]]
+
+                        elif msg_type == "token":
+                            # Stream token to UI
+                            token = data.get("token", "")
+                            accumulated_response += token
+                            # Yield updated history with partial response
+                            status_prefix = f"*Intent: {intent}*\n\n" if intent else ""
+                            yield history + [[user_message, status_prefix + accumulated_response]]
+
+                        elif msg_type == "citations":
+                            citations = data.get("citations", [])
+
+                        elif msg_type == "done":
+                            latency_ms = data.get("latency_ms")
+                            logging.info(f"WebSocket stream completed in {latency_ms}ms")
+                            break
+
+                        elif msg_type == "error":
+                            error_msg = data.get("error", "Unknown error")
+                            yield history + [[user_message, f"ERROR: Error: {error_msg}"]]
+                            ws.close()
+                            return
+
+                    except websocket.WebSocketTimeoutException:
+                        logging.warning("WebSocket timeout")
+                        break
+                    except json.JSONDecodeError as e:
+                        logging.error(f"Failed to parse WebSocket message: {e}")
+                        continue
+
+                ws.close()
+
+                # Add citations to final response
+                if citations:
+                    accumulated_response += "\n\n**Citations:**\n"
+                    for cite in citations:
+                        title = cite.get("title", "Unknown")
+                        grant_id = cite.get("grant_id", "N/A")
+                        accumulated_response += f"- **{title}** (ID: {grant_id})\n"
+
+                # Add intent badge if available
+                if intent:
+                    final_response = f"*Intent: {intent}*\n\n{accumulated_response}"
+                else:
+                    final_response = accumulated_response
+
+                yield history + [[user_message, final_response]]
+
+            except Exception as e:
+                logging.error(f"WebSocket streaming error: {e}")
+                yield history + [[user_message, f"ERROR: WebSocket error: {e}"]]
+            return
+
+        if use_sse and httpx:
+            # Use SSE streaming from API endpoint
+            try:
+                accumulated_response = ""
+                citations = []
+
+                with httpx.Client(timeout=60.0) as client:
+                    with client.stream(
+                        "POST",
+                        "http://localhost:8000/qa/stream",  # Adjust URL as needed
+                        json={"query": user_message, "use_llm_routing": True}
+                    ) as response:
+                        for line in response.iter_lines():
+                            if line.startswith("data: "):
+                                data_str = line[6:]  # Remove "data: " prefix
+                                try:
+                                    data = json.loads(data_str)
+                                    event_type = data.get("type")
+
+                                    if event_type == "token":
+                                        token = data.get("token", "")
+                                        accumulated_response += token
+                                        # Yield updated history with partial response
+                                        yield history + [[user_message, accumulated_response]]
+
+                                    elif event_type == "citations":
+                                        citations = data.get("citations", [])
+
+                                    elif event_type == "error":
+                                        error = data.get("error", "Unknown error")
+                                        yield history + [[user_message, f"ERROR: Error: {error}"]]
+                                        return
+
+                                except json.JSONDecodeError:
+                                    continue
+
+                # Add citations to final response
+                if citations:
+                    accumulated_response += "\n\n**Citations:**\n"
+                    for cite in citations:
+                        accumulated_response += f"- {cite.get('title')} (ID: {cite.get('grant_id')})\n"
+
+                yield history + [[user_message, accumulated_response]]
+
+            except Exception as e:
+                logging.error(f"SSE streaming error: {e}")
+                yield history + [[user_message, f"ERROR: Streaming error: {e}"]]
+            return
+
+        # Fallback to original non-streaming chat
+        response, updated_history = self.chat(user_message, history)
+        yield updated_history
+
+    def chat(self, user_message: str, history: List) -> Tuple[str, List]:
+        """
+        Process a chat message (non-streaming).
+
+        Args:
+            user_message: User's input
+            history: Gradio chat history
+
+        Returns:
+            (response, updated_history) tuple
+        """
+        import time
+
+        if not self.initialized:
+            return "WARNING: System not initialized. Please restart the app.", history
+
+        start_time = time.time()
+        tools_called = []
+        timing_info = {}
+
+        try:
+            # Add user message
+            self.messages.append({"role": "user", "content": user_message})
+
+            # Call LLM with function calling
+            llm_start = time.time()
+            response = self.llm_client.client.chat.completions.create(
+                model=self.llm_client.model,
+                messages=self.messages,
+                tools=self.available_tools,
+                tool_choice="auto",
+                temperature=0.5,  # INCREASED from 0.1 to allow more thorough, creative responses
+                max_tokens=4096,  # INCREASED to allow detailed summaries without truncation
+            )
+            timing_info["llm_call"] = time.time() - llm_start
+
+            response_message = response.choices[0].message
+            tool_calls = response_message.tool_calls
+
+            # If LLM wants to call tools
+            if tool_calls:
+                # Add assistant's response with tool calls
+                self.messages.append(response_message)
+
+                # Execute each tool call
+                tools_start = time.time()
+                for tool_call in tool_calls:
+                    function_name = tool_call.function.name
+                    import json
+                    function_args = json.loads(tool_call.function.arguments)
+
+                    logging.info(f"Calling: {function_name}({function_args})")
+                    tools_called.append(function_name)  # Track for logging
+
+                    # Execute tool
+                    tool_start = time.time()
+                    tool_result = self._dispatch_tool(function_name, function_args)
+                    tool_time = time.time() - tool_start
+                    logging.info(f"{function_name} took {tool_time:.2f}s")
+                    timing_info[f"tool_{function_name}"] = tool_time
+
+                    # Add tool result
+                    self.messages.append({
+                        "role": "tool",
+                        "tool_call_id": tool_call.id,
+                        "name": function_name,
+                        "content": str(tool_result)
+                    })
+
+                # Get final response
+                final_start = time.time()
+                final_response = self.llm_client.client.chat.completions.create(
+                    model=self.llm_client.model,
+                    messages=self.messages,
+                    temperature=0.5,  # INCREASED from 0.1 for thorough final responses
+                    max_tokens=4096,  # INCREASED to allow complete answers without truncation
+                )
+                timing_info["final_llm_call"] = time.time() - final_start
+
+                assistant_message = final_response.choices[0].message.content
+                self.messages.append({"role": "assistant", "content": assistant_message})
+            else:
+                # No tool calls
+                assistant_message = response_message.content
+                self.messages.append({"role": "assistant", "content": assistant_message})
+
+            # Log timing info
+            response_time_ms = int((time.time() - start_time) * 1000)
+            timing_str = " | ".join([f"{k}:{v:.2f}s" for k, v in timing_info.items()])
+            logging.info(f"Total: {response_time_ms}ms | {timing_str}")
+
+            # Direct logging to CSV (simpler, more reliable)
+            try:
+                import csv
+                from datetime import datetime
+
+                log_dir = Path(".") / "logs"
+                log_dir.mkdir(parents=True, exist_ok=True)
+
+                today = datetime.now().strftime("%Y%m%d")
+                csv_path = log_dir / f"queries_{today}.csv"
+
+                # Check if file exists to write header
+                file_exists = csv_path.exists()
+
+                with open(csv_path, 'a', newline='', encoding='utf-8') as f:
+                    writer = csv.writer(f)
+
+                    # Write header if new file
+                    if not file_exists:
+                        writer.writerow([
+                            'timestamp', 'user_query', 'ai_response', 'tools_called',
+                            'response_time_ms', 'success', 'rating', 'feedback', 'model'
+                        ])
+
+                    # Write the query
+                    writer.writerow([
+                        datetime.now().isoformat(),
+                        user_message,
+                        assistant_message[:5000],  # Truncate very long responses
+                        ','.join(tools_called),
+                        response_time_ms,
+                        'True',
+                        '',  # rating
+                        '',  # feedback
+                        self.llm_client.model
+                    ])
+
+                logging.info(f"Query logged to {csv_path}")
+
+            except Exception as log_error:
+                logging.error(f"ERROR: Logging failed: {log_error}", exc_info=True)
+
+            return assistant_message, history + [[user_message, assistant_message]]
+
+        except Exception as e:
+            error_msg = f"ERROR: Error: {e}"
+            logging.error(f"Chat error: {e}", exc_info=True)
+
+            # Log failed interactions too
+            response_time_ms = int((time.time() - start_time) * 1000)
+
+            # Direct logging to CSV (simpler, more reliable)
+            try:
+                import csv
+                from datetime import datetime
+
+                log_dir = Path(".") / "logs"
+                log_dir.mkdir(parents=True, exist_ok=True)
+
+                today = datetime.now().strftime("%Y%m%d")
+                csv_path = log_dir / f"queries_{today}.csv"
+
+                # Check if file exists to write header
+                file_exists = csv_path.exists()
+
+                with open(csv_path, 'a', newline='', encoding='utf-8') as f:
+                    writer = csv.writer(f)
+
+                    # Write header if new file
+                    if not file_exists:
+                        writer.writerow([
+                            'timestamp', 'user_query', 'ai_response', 'tools_called',
+                            'response_time_ms', 'success', 'rating', 'feedback', 'model'
+                        ])
+
+                    # Write the failed query
+                    writer.writerow([
+                        datetime.now().isoformat(),
+                        user_message,
+                        error_msg,
+                        ','.join(tools_called),
+                        response_time_ms,
+                        'False',
+                        '',  # rating
+                        '',  # feedback
+                        self.llm_client.model
+                    ])
+
+                logging.info(f"Failed query logged to {csv_path}")
+
+            except Exception as log_error:
+                logging.error(f"ERROR: Logging failed: {log_error}", exc_info=True)
+
+            return error_msg, history + [[user_message, error_msg]]
+
+
+def create_demo_ui(demo: GrantAnalystDemo) -> gr.Blocks:
+    """Create the Gradio UI."""
+
+    with gr.Blocks(
+        title="Grant Analyst Demo",
+        theme=gr.themes.Soft(),
+        css="""
+        .contain { max-width: 1200px; margin: auto; }
+        #status-box { padding: 1rem; border-radius: 8px; margin-bottom: 1rem; }
+        """
+    ) as app:
+
+        gr.Markdown(
+            """
+            # Grant Analyst — Client Demo
+
+            **Intelligent grant search and analysis powered by AI**
+
+            Ask questions about Innovate UK grants in natural language, or try one of the preset questions below.
+            """
+        )
+
+        # Status display
+        status = gr.Markdown(
+            "Initializing system...",
+            elem_id="status-box"
+        )
+
+        # Initialize on load
+        def init_system():
+            success, msg = demo.initialize()
+            if success:
+                return f"**System Ready** — {msg}"
+            else:
+                return f"ERROR: **Initialization Failed** — {msg}"
+
+        # Chatbot interface
+        with gr.Row():
+            with gr.Column(scale=3):
+                chatbot = gr.Chatbot(
+                    label="Conversation",
+                    height=500,
+                    bubble_full_width=False
+                )
+
+                with gr.Row():
+                    msg_input = gr.Textbox(
+                        label="Your Question",
+                        placeholder="Ask me anything about grants...",
+                        scale=4
+                    )
+                    send_btn = gr.Button("Send", scale=1, variant="primary")
+
+                with gr.Row():
+                    clear_btn = gr.Button("Clear Conversation")
+
+            with gr.Column(scale=1):
+                gr.Markdown("### Preset Questions")
+                gr.Markdown("*Click to try these guaranteed examples:*")
+
+                preset_btns = []
+                for label, question in PRESET_QUESTIONS.items():
+                    btn = gr.Button(label, size="sm")
+                    preset_btns.append((btn, question))
+
+        # Examples section
+        gr.Markdown(
+            """
+            ---
+            ### What You Can Ask
+
+            **Listing & Search:**
+            - "Show me all battery-related grants"
+            - "What grants are available for SMEs?"
+            - "List grants with funding over £500k"
+
+            **Deadlines:**
+            - "What are the upcoming deadlines?"
+            - "When does competition-2313 close?"
+
+            **Comparisons:**
+            - "Compare competition-2313 and competition-2314"
+            - "What's the difference between these two grants?"
+
+            **Details:**
+            - "Tell me about competition-2317"
+            - "Summarize the battery feasibility grant"
+            """
+        )
+
+        # Event handlers
+        def respond(message, chat_history):
+            """Handle user message."""
+            # Use non-streaming response
+            bot_response, updated_history = demo.chat(message, chat_history)
+            yield "", updated_history
+
+        def use_preset(question, chat_history):
+            """Handle preset question click."""
+            for result in respond(question, chat_history):
+                yield result
+
+        # Wire up events
+        msg_input.submit(
+            respond,
+            [msg_input, chatbot],
+            [msg_input, chatbot]
+        )
+        send_btn.click(
+            respond,
+            [msg_input, chatbot],
+            [msg_input, chatbot]
+        )
+        clear_btn.click(lambda: [], None, chatbot)
+
+        for btn, question in preset_btns:
+            btn.click(
+                use_preset,
+                inputs=[gr.State(question), chatbot],
+                outputs=[msg_input, chatbot]
+            )
+
+        # Initialize on load
+        app.load(init_system, None, status)
+
+    return app
+
+
+def main():
+    """Launch the demo application."""
+
+    parser = argparse.ArgumentParser(description="Grant Analyst Demo App")
+    parser.add_argument("--share", action="store_true", help="Create public shareable link")
+    parser.add_argument("--port", type=int, default=7860, help="Port to run on")
+    args = parser.parse_args()
+
+    # Setup logging (create log directory if needed)
+    log_handlers = [logging.StreamHandler(sys.stderr)]
+    try:
+        from pathlib import Path
+        log_dir = Path("_out/logs")
+        log_dir.mkdir(parents=True, exist_ok=True)
+        log_handlers.append(logging.FileHandler(log_dir / "demo_app.log"))
+    except Exception as e:
+        logging.warning(f"Could not create log file: {e}")
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        handlers=log_handlers
+    )
+
+    # Create demo instance
+    demo = GrantAnalystDemo()
+
+    # Create and launch UI
+    app = create_demo_ui(demo)
+
+    print("\n" + "="*60)
+    print("Launching Grant Analyst Demo...")
+    print("="*60)
+
+    app.launch(
+        server_name="0.0.0.0",
+        server_port=args.port,
+        share=args.share,
+        show_error=True,
+    )
+
+
+if __name__ == "__main__":
+    main()

analyzer/chat/insight_tools.py

@@ -0,0 +1,94 @@
+# src/analyzer/chat/insight_tools.py
+from __future__ import annotations
+from typing import Dict, Any, List, Optional
+
+from ..config import load_config
+try:
+    from ..llm_client import LLMClient
+except Exception:
+    LLMClient = None  # type: ignore
+
+try:
+    from ..search.hybrid_index import load_index, search, top_supporting_for_grant
+except Exception:
+    load_index = None  # type: ignore
+    search = None     # type: ignore
+    top_supporting_for_grant = None  # type: ignore
+
+from ..prompt_templates import build_open_prompt
+
+class InsightTools:
+    """
+    Retrieval + LLM glue for open, lightly-guarded answers.
+    - Accepts an existing LLMClient (preferred) to ensure identical config across entrypoints
+    - Falls back gracefully if LLM or index is unavailable
+    """
+
+    def __init__(self, llm_client: Optional["LLMClient"] = None):
+        self._cfg = load_config()
+        self._client = llm_client
+        if self._client is None and LLMClient is not None:
+            try:
+                self._client = LLMClient(self._cfg)
+            except Exception:
+                self._client = None
+
+        self._idx = None
+        if callable(load_index):
+            try:
+                self._idx = load_index()
+            except Exception:
+                self._idx = None
+
+    # Optional helper some callers use to append snippets under a summary
+    def supporting_snippets_md(self, grant_id: str, k: int = 5) -> str:
+        if not self._idx or not callable(top_supporting_for_grant):
+            return ""
+        try:
+            hits = top_supporting_for_grant(self._idx, grant_id, k=k)
+        except Exception:
+            return ""
+        items: List[str] = []
+        for doc, score in hits:
+            if doc.get("_source") != "supporting":
+                continue
+            sec = doc.get("section") or "(Supporting)"
+            url = doc.get("url","")
+            txt = (doc.get("text","") or "").replace("\n"," ")
+            snippet = (txt[:400] + "…") if len(txt) > 400 else txt
+            items.append(f"- **{sec}** — {url}\n  > {snippet}")
+        if not items:
+            return ""
+        return "\n\n---\n**Supporting info (top snippets)**\n" + "\n".join(items)
+
+    def insight_search(self, question: str, *, k: int = 8, use_llm: bool = True) -> Dict[str, Any]:
+        """Open-style grounded QA: supply question + top snippets; let the LLM pick format and length."""
+        if not self._idx or not callable(search):
+            return {"answer_md": "Search index not available."}
+
+        hits = search(self._idx, question, k=k, filters=None)
+        ev_lines: List[str] = []
+        raw_context: List[str] = []
+        for doc, score in hits[:k]:
+            sec = doc.get("section") or doc.get("title","")
+            url = doc.get("url","")
+            txt = (doc.get("text","") or "")
+            ev_lines.append(f"- **{sec}** — {url}")
+            raw_context.append(f"[{sec}] {url}\n{txt}")
+
+        evidence_md = "**Sources**\n" + "\n".join(ev_lines) if ev_lines else ""
+
+        if use_llm and self._client:
+            payload = build_open_prompt(
+                provider=getattr(self._cfg, "provider", "openai") if self._cfg else "openai",
+                question=question,
+                context="\n\n---\n".join(raw_context[:6]),
+            )
+            try:
+                # Slightly higher budget; model decides format naturally
+                answer = self._client.chat(payload["messages"], max_tokens=1200, temperature=0.3)
+                return {"answer_md": f"{answer}\n\n---\n{evidence_md}"}
+            except Exception:
+                pass
+
+        return {"answer_md": "No LLM available. See sources below.\n\n---\n" + evidence_md}

analyzer/chat/memory.py

@@ -0,0 +1,95 @@
+# memory.py — lightweight conversation memory
+from __future__ import annotations
+from dataclasses import dataclass, asdict
+from typing import List, Dict, Optional
+import json, time, os
+
+@dataclass
+class Turn:
+    role: str          # "user" | "assistant" | "tool"
+    content: str
+    meta: Dict = None
+    ts: float = 0.0
+
+    def to_dict(self):
+        d = asdict(self)
+        d["ts"] = self.ts or time.time()
+        d["meta"] = self.meta or {}
+        return d
+
+class ConversationMemory:
+    """
+    Simple memory with:
+      - buffer: last N turns (short-term)
+      - summary: rolling abstractive summary (long-term)
+      - entities: key entities/ids spotted so far
+    """
+    def __init__(self, path: str, buffer_size: int = 12):
+        self.path = path
+        self.buffer_size = buffer_size
+        self.buffer: List[Turn] = []
+        self.summary: str = ""
+        self.entities: Dict[str, List[str]] = {}  # e.g., {"competition_id": ["2313", ...]}
+        self._load()
+
+    # ------------ persistence ------------
+    def _load(self):
+        if not os.path.exists(self.path): return
+        with open(self.path, "r") as f:
+            data = json.load(f)
+        self.buffer = [Turn(**t) for t in data.get("buffer", [])]
+        self.summary = data.get("summary", "")
+        self.entities = data.get("entities", {})
+
+    def _save(self):
+        os.makedirs(os.path.dirname(self.path), exist_ok=True)
+        with open(self.path, "w") as f:
+            json.dump({
+                "buffer": [t.to_dict() for t in self.buffer],
+                "summary": self.summary,
+                "entities": self.entities,
+            }, f, ensure_ascii=False, indent=2)
+
+    # ------------ public API ------------
+    def add_turn(self, role: str, content: str, meta: Optional[Dict]=None):
+        self.buffer.append(Turn(role=role, content=content, meta=meta or {}, ts=time.time()))
+        if len(self.buffer) > self.buffer_size:
+            self.buffer = self.buffer[-self.buffer_size:]
+        self._save()
+
+    def get_context(self) -> Dict:
+        """What to feed into prompts/tools."""
+        return {
+            "summary": self.summary,
+            "recent": [{"role": t.role, "content": t.content} for t in self.buffer[-self.buffer_size:]],
+            "entities": self.entities,
+        }
+
+    def update_summary(self, llm_summarize_fn):
+        """
+        Call with a function that maps (summary, recent) -> new_summary.
+        Only do this occasionally (e.g., every 6–10 user turns).
+        """
+        if not self.buffer:
+            return
+        recent_text = "\n".join(
+            f"{t.role.upper()}: {t.content}" for t in self.buffer[-self.buffer_size:]
+        )
+        prompt = (
+            "You are a diligent note-taker. Update the long-term summary of this conversation.\n"
+            "Keep it under 150 words. Capture tasks, preferences, important grant IDs/themes, and open questions.\n\n"
+            f"EXISTING SUMMARY:\n{self.summary or '(none)'}\n\n"
+            f"RECENT TURNS:\n{recent_text}\n\n"
+            "Return ONLY the updated summary text."
+        )
+        new_sum = llm_summarize_fn(prompt).strip()
+        if new_sum:
+            self.summary = new_sum
+            self._save()
+
+    def add_entity(self, kind: str, value: str):
+        if not value: return
+        arr = self.entities.setdefault(kind, [])
+        if value not in arr:
+            arr.append(value)
+            self._save()

analyzer/chat/query_router.py

@@ -0,0 +1,345 @@
+# src/analyzer/chat/query_router.py
+"""
+Query routing: detect user intent from natural language questions.
+
+Supports synonyms and variations:
+- "What grants are available?" = "List all grants"
+- "Show me funding opportunities" = "List all grants"
+- "List all open grants" = "Filter by status=open"
+- "What can I apply for?" = "List available opportunities"
+
+This helps reduce redundant information requests and improves UX.
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Dict, Optional, Tuple, List
+import json, re, hashlib, time, logging
+
+logger = logging.getLogger(__name__)
+
+_INTENTS = {"list", "summarize", "compare", "deadlines", "search", "general", "get_grant", "list_grants"}
+
+# Routing cache with 24-hour TTL
+_routing_cache: Dict[str, Tuple[Dict, float]] = {}
+_CACHE_TTL = 86400  # 24 hours in seconds
+
+# Accept "competition-2315", "2315", "comp-2315"
+_ID_RE = re.compile(r"(?:comp(?:etition)?-)?([0-9]{3,7})", re.IGNORECASE)
+
+# Synonym groups for better intent detection
+GRANT_SYNONYMS = {
+    "grants", "grant", "calls", "call", "opportunities", "opportunity",
+    "funding", "competitions", "competition", "schemes", "scheme"
+}
+
+LIST_INTENT_SYNONYMS = {
+    "list", "show", "display", "what", "which", "all", "available",
+    "open", "upcoming", "closed", "find", "get"
+}
+
+STOPWORDS = {
+    "what","which","are","is","the","a","an","for","about","of","to","in","on","and","with",
+    "available","there","any","please","show","me","find","search","grants","grant","calls",
+    "opportunities","funding","compare","vs","versus","between","two","both",
+    "can", "i", "me", "my", "your", "apply", "get"
+}
+
+@dataclass
+class Routed:
+    intent: str
+    args: Dict
+    confidence: float
+    def to_dict(self) -> Dict:
+        return {"intent": self.intent, "args": self.args, "confidence": self.confidence}
+
+def _extract_ids(text: str) -> Tuple[List[str], str]:
+    ids = [m.group(1) for m in _ID_RE.finditer(text)]
+    residual = _ID_RE.sub("", text).strip()
+    return ids, residual
+
+def _keywords_from_question(t: str) -> str:
+    # pull phrase after 'for ' or 'in ' if present, else keep content tokens
+    m = re.search(r"(?:for|in)\s+([A-Za-z0-9\- ][A-Za-z0-9\-\s]+)\??$", t, flags=re.IGNORECASE)
+    phrase = (m.group(1) if m else t).strip()
+    tokens = [w.lower() for w in re.findall(r"[A-Za-z0-9\-]+", phrase) if w.lower() not in STOPWORDS]
+    return " ".join(tokens[-4:]) if tokens else phrase.lower()
+
+def _detect_list_intent(text: str) -> bool:
+    """
+    Detect if user wants to list/view all grants.
+
+    Recognizes patterns like:
+    - "What grants are available?"
+    - "Show me funding opportunities"
+    - "List all open grants"
+    - "What can I apply for?"
+    """
+    low = text.lower()
+
+    # Explicit list/show commands
+    if low.startswith(("list", "show", "display", "get me")):
+        return True
+
+    # "What/which ... grants/opportunities/funding" patterns
+    list_patterns = [
+        "what grants", "what funding", "what opportunities", "what calls",
+        "which grants", "which funding", "which opportunities", "which calls",
+        "what can i apply for", "what's available", "what's open",
+        "show me grants", "show me funding", "show me opportunities", "show me calls",
+        "what opportunities", "what calls"
+    ]
+    if any(p in low for p in list_patterns):
+        return True
+
+    # "All grants" or "all open/closed grants"
+    if "all " in low and any(w in low for w in GRANT_SYNONYMS):
+        return True
+
+    return False
+
+
+def _detect_status_filter(text: str) -> Optional[str]:
+    """Extract status filter from question if present."""
+    low = text.lower()
+
+    if "open" in low and ("grants" in low or "opportunities" in low or "calls" in low):
+        return "open"
+    if "closed" in low and any(w in low for w in GRANT_SYNONYMS):
+        return "closed"
+    if "upcoming" in low and any(w in low for w in GRANT_SYNONYMS):
+        return "upcoming"
+
+    return None
+
+
+def classify_complexity(query: str) -> str:
+    """
+    Classify query complexity to select appropriate model.
+
+    Returns:
+        'simple': Use gpt-5-mini for simple translation/explanation tasks
+        'complex': Use gpt-5 for detailed analysis/comparisons
+        'medium': Default, use gpt-5-mini
+    """
+    low = query.lower()
+
+    # Simple queries: translation, explanation, layman's terms
+    simple_keywords = {"translate", "explain", "simple", "layman", "what is", "define"}
+    if any(kw in low for kw in simple_keywords):
+        return 'simple'
+
+    # Complex queries: detailed analysis, comparisons
+    complex_keywords = {"compare", "analyze", "detailed", "comprehensive", "in-depth"}
+    if any(kw in low for kw in complex_keywords):
+        return 'complex'
+
+    # Default to medium complexity
+    return 'medium'
+
+
+def _route_with_llm(text: str) -> Optional[str]:
+    """
+    Route query using GPT-5-nano for intelligent intent classification.
+
+    Returns:
+        Intent string or None if LLM fails
+    """
+    try:
+        from ..llm_client import LLMClient
+        from ..config import load_config
+
+        # Initialize LLM client with router model
+        config = load_config()
+        client = LLMClient(config)
+
+        if not client.is_ready():
+            return None
+
+        # Build routing prompt
+        prompt = f"""Classify this query into ONE of these intents: search, summarize, compare, get_grant, list_grants, deadlines, general.
+
+Query: {text}
+
+Respond with ONLY the intent word, nothing else."""
+
+        # Get LLM classification with routing parameters
+        messages = [
+            {"role": "system", "content": "You are a query intent classifier. Respond with only one word."},
+            {"role": "user", "content": prompt}
+        ]
+
+        response = client.chat(
+            messages,
+            model_type="router",  # Use gpt-5-nano
+            verbosity="low",
+            reasoning_effort="minimal",
+            max_tokens=10,
+            temperature=0.1
+        )
+
+        # Parse single word response
+        intent = response.strip().lower()
+
+        # Validate intent
+        if intent in _INTENTS:
+            logger.info(f"🤖 LLM routing: '{text[:50]}...' -> {intent}")
+            return intent
+        else:
+            logger.warning(f"LLM returned invalid intent: {intent}")
+            return None
+
+    except Exception as e:
+        logger.warning(f"LLM routing failed: {e}")
+        return None
+
+
+def _route_with_regex(text: str) -> Dict:
+    """
+    Fallback regex-based routing (original implementation).
+
+    Returns:
+        Routed dict with intent, args, and confidence
+    """
+    t = text.strip()
+    low = t.lower()
+
+    # Check for status-specific listing first (higher priority)
+    status_filter = _detect_status_filter(t)
+
+    # Explicit list/show commands (highest confidence)
+    if low.startswith("list") or low.startswith("show") or _detect_list_intent(t):
+        # Extract keyword, but be smart about filler words
+        kw = t.split(" ", 1)[1].strip() if " " in t else ""
+
+        # Remove punctuation
+        kw = re.sub(r'[?!.,;:]', '', kw).strip()
+
+        # Clean up filler words
+        filler = {
+            "me", "please", "show", "list", "all", "available", "open", "closed", "upcoming",
+            "grants", "grant", "opportunities", "opportunity", "funding", "calls", "call",
+            "is", "are", "the", "a", "an", "and", "or", "for", "to", "in", "on", "with"
+        }
+        kw_tokens = [w for w in kw.lower().split() if w not in filler]
+        kw = " ".join(kw_tokens) if kw_tokens else ""
+
+        args = {}
+        if kw:
+            args["keyword"] = kw
+        if status_filter:
+            args["status"] = status_filter
+
+        args["limit"] = None
+        return Routed("list", args, 0.75).to_dict()
+
+    if low.startswith("summarize") or low.startswith("summarise"):
+        ids, _ = _extract_ids(t)
+        if len(ids) >= 2:
+            return Routed("compare", {"grant_id_a": f"competition-{ids[0]}", "grant_id_b": f"competition-{ids[1]}"}, 0.85).to_dict()
+        if len(ids) == 1:
+            return Routed("summarize", {"grant_id": f"competition-{ids[0]}"}, 0.85).to_dict()
+        kw = t.split(" ", 1)[1].strip() if " " in t else ""
+        return Routed("search", {"keyword": kw, "limit": None}, 0.6).to_dict()
+
+    # Deadline queries
+    if any(p in low for p in ["deadline", "close date", "when is it due", "when do i need to apply", "application deadline"]):
+        return Routed("deadlines", {"n": None}, 0.75).to_dict()
+
+    # compare / vs / versus / between → compare two grants
+    if "compare" in low or " vs " in low or "versus" in low or "between" in low:
+        ids, residual = _extract_ids(t)
+        if len(ids) >= 2:
+            return Routed(
+                "compare",
+                {"grant_id_a": f"competition-{ids[0]}", "grant_id_b": f"competition-{ids[1]}"},
+                0.82
+            ).to_dict()
+        if len(ids) == 1:
+            return Routed("summarize", {"grant_id": f"competition-{ids[0]}"}, 0.7).to_dict()
+
+    # Natural search
+    if any(w in low for w in ["grant", "funding", "competition", "call", "apply", "what", "which", "find", "search"]):
+        kw = _keywords_from_question(t)
+        return Routed("search", {"keyword": kw, "limit": None}, 0.65).to_dict()
+
+    return Routed("general", {"question": t}, 0.5).to_dict()
+
+
+def route(text: str, *, use_llm: bool = True) -> Dict:
+    """
+    Route a user query to the appropriate intent handler.
+
+    Uses GPT-5-nano for intelligent routing with regex fallback.
+    Results are cached for 24 hours for performance.
+
+    Args:
+        text: User query text
+        use_llm: If True, use GPT-5-nano for routing (default: True)
+
+    Returns:
+        Dict with intent, args, and confidence
+    """
+    t = text.strip()
+
+    # Check cache first
+    cache_key = hashlib.md5(t.lower().encode()).hexdigest()
+    if cache_key in _routing_cache:
+        cached_result, cached_time = _routing_cache[cache_key]
+        if time.time() - cached_time < _CACHE_TTL:
+            logger.debug(f"📦 Cache HIT for routing: '{t[:50]}...'")
+            return cached_result
+
+    # Try LLM routing first (if enabled)
+    result = None
+    if use_llm:
+        llm_intent = _route_with_llm(t)
+        if llm_intent:
+            # Extract args based on intent
+            args = {}
+            ids, _ = _extract_ids(t)
+
+            if llm_intent == "summarize" and len(ids) == 1:
+                args["grant_id"] = f"competition-{ids[0]}"
+            elif llm_intent == "compare" and len(ids) >= 2:
+                args["grant_id_a"] = f"competition-{ids[0]}"
+                args["grant_id_b"] = f"competition-{ids[1]}"
+            elif llm_intent in ("search", "list_grants"):
+                kw = _keywords_from_question(t)
+                if kw:
+                    args["keyword"] = kw
+                status = _detect_status_filter(t)
+                if status:
+                    args["status"] = status
+                args["limit"] = None
+            elif llm_intent == "get_grant" and len(ids) == 1:
+                args["grant_id"] = f"competition-{ids[0]}"
+            elif llm_intent == "deadlines":
+                args["n"] = None
+            else:
+                args["question"] = t
+
+            result = Routed(llm_intent, args, 0.95).to_dict()
+
+    # Fall back to regex if LLM failed or disabled
+    if result is None:
+        logger.debug(f"🔧 Using regex fallback for: '{t[:50]}...'")
+        result = _route_with_regex(t)
+
+    # Cache the result
+    _routing_cache[cache_key] = (result, time.time())
+
+    return result
+
+# Self-test
+if __name__ == "__main__":
+    tests = [
+        "list AI calls",
+        "summarize competition-2316",
+        "summarize 2315 2318",
+        "compare 2315 vs 2318 for total funding per project and start-by",
+        "find grants for battery feasibility studies",
+        "what funding is available for hydrogen?",
+        "when is the deadline?",
+    ]
+    for s in tests:
+        print(s, "->", route(s))

analyzer/chat/run_chat.py

@@ -0,0 +1,408 @@
+"""
+run_chat.py — unified interactive terminal chatbot for grant data
+
+Features (controlled by flags):
+- Startup diagnostics (--verbose)
+- LLM-assisted routing (--use-llm-routing, default ON)
+- Memory persistence (--with-memory)
+- Extended tools (auto-detected or --extended-tools)
+
+Usage:
+  python -m src.analyzer.chat.run_chat
+  python -m src.analyzer.chat.run_chat --verbose --limit 10
+  python -m src.analyzer.chat.run_chat --no-llm-routing
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+import time
+from collections import Counter
+from pathlib import Path
+from typing import List, Dict, Optional
+import re
+
+from dotenv import load_dotenv; load_dotenv()
+
+from ..config import load_config
+from ..data_loader import load_current_grants, load_past_winners
+from ..llm_client import LLMClient
+from .chat_tools import ChatTools
+from .query_router import route
+from ..logging_setup import setup_logging
+from ..telemetry.logger import QALogger
+from .tool_schemas import openai_tools, detect_extended_features
+from ..utils.errors import (
+    GrantAnalyzerError,
+    ValidationError,
+    DataLoadError,
+    SearchError,
+    LLMError,
+    ConfigError
+)
+
+# Optional: memory (graceful if not available)
+try:
+    from .memory import ConversationMemory
+    MEMORY_AVAILABLE = True
+except ImportError:
+    MEMORY_AVAILABLE = False
+
+
+# ---------------- Domain-term extraction (for dynamic themes) ---------------- #
+
+_STOP = {
+    "the","a","an","and","or","of","for","to","in","on","with","by","about","into","from","at","as",
+    "call","grant","competition","innovate","uk","round","study","studies","feasibility","phase",
+    "funding","programme","program","projects","project","research","development","pilot"
+}
+_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9\-]+", re.IGNORECASE)
+
+
+def _extract_domain_terms(rows: List[Dict], top_n: int = 150) -> set[str]:
+    """
+    Build a compact vocabulary from your grant corpus (titles/summaries/themes).
+    This feeds the router's dynamic theme detection (no hardcoding).
+    """
+    texts = []
+    for r in rows:
+        parts = [
+            str(r.get("title","")),
+            str(r.get("summary","")),
+            str(r.get("overview","")),
+            str(r.get("scope","")),
+            str(r.get("theme","")),
+        ]
+        texts.append(" ".join(p for p in parts if p))
+
+    unigrams = Counter()
+    bigrams  = Counter()
+
+    for txt in texts:
+        toks = [t.lower() for t in _TOKEN_RE.findall(txt) if t.lower() not in _STOP and len(t) >= 3]
+        unigrams.update(toks)
+        for i in range(len(toks)-1):
+            w1, w2 = toks[i], toks[i+1]
+            if w1 in _STOP or w2 in _STOP:
+                continue
+            bigrams.update([f"{w1} {w2}"])
+
+    vocab = set([w for w, _ in unigrams.most_common(top_n)])
+    vocab |= set([w for w, _ in bigrams.most_common(max(1, top_n // 2))])
+    return vocab
+
+
+def _startup_diagnostics(cfg, *, llm_ok: bool, idx_ok: bool, mem_ok: bool) -> str:
+    """Generate startup diagnostics string."""
+    provider = getattr(cfg, "provider", "?") if hasattr(cfg, "provider") else cfg.get("provider", "?")
+    model = getattr(cfg, "model", "?") if hasattr(cfg, "model") else cfg.get("model", "?")
+    openai_key_present = bool(os.getenv("OPENAI_API_KEY"))
+
+    lines = [
+        "=== Grant Analyst Chat — Startup Diagnostics ===",
+        f"Provider:   {provider}",
+        f"Model:      {model}",
+        f"LLM Ready:  {llm_ok}",
+        f"Index OK:   {idx_ok}",
+        f"Memory OK:  {mem_ok}",
+        f"API Key:    {'✓' if openai_key_present else '✗'}",
+        "",
+    ]
+    return "\n".join(lines)
+
+
+def _index_ok_verbose() -> tuple[bool, str]:
+    """Check if hybrid index is present and valid."""
+    from pathlib import Path
+    import pickle
+
+    p = Path("data/index/hybrid_index.pkl")
+    if not p.exists():
+        return False, f"missing file: {p}"
+
+    try:
+        with p.open("rb") as f:
+            payload = pickle.load(f)
+    except Exception as e:
+        return False, f"could not read {p.name}: {e}"
+
+    if not isinstance(payload, dict):
+        return False, f"{p.name} is not a dict payload"
+
+    docs = payload.get("docs")
+    if not isinstance(docs, list):
+        return False, f"{p.name} has no 'docs' list"
+
+    if len(docs) == 0:
+        return False, f"{p.name} contains 0 docs"
+
+    return True, f"{p.name} with {len(docs)} docs"
+
+
+def main(argv: List[str] | None = None) -> None:
+    setup_logging()
+    cfg = load_config()
+
+    ap = argparse.ArgumentParser(description="Interactive chatbot for grant insights")
+    ap.add_argument("--snapshots-dir", type=Path, default=Path("data/snapshots"))
+    ap.add_argument("--history-xlsx", type=Path,
+                   default=Path("data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx"))
+    ap.add_argument("--limit", type=int, default=0,
+                   help="Load at most N grants (0 = all)")
+    ap.add_argument("--log-jsonl", type=Path, default=Path("_out/chat.jsonl"))
+
+    # Feature flags
+    ap.add_argument("--use-llm-routing", dest="use_llm_routing", action="store_true", default=True,
+                   help="Enable LLM-assisted routing (default: ON)")
+    ap.add_argument("--no-llm-routing", dest="use_llm_routing", action="store_false",
+                   help="Disable LLM routing (heuristics only)")
+    ap.add_argument("--extended-tools", action="store_true", default=False,
+                   help="Force extended tools (insight_search, fetch_link)")
+    ap.add_argument("--with-memory", action="store_true", default=False,
+                   help="Enable conversation memory (requires memory.py)")
+    ap.add_argument("--verbose", action="store_true", default=False,
+                   help="Show startup diagnostics")
+
+    args = ap.parse_args(argv)
+
+    # Load data
+    logging.info("Loading data ...")
+    current = load_current_grants(args.snapshots_dir, limit=args.limit or None)
+    past = load_past_winners(args.history_xlsx)
+    logging.info("Loaded %d current grants; %d past winners", len(current), len(past))
+
+    # Initialize components
+    tools = ChatTools(current, past)
+
+    llm_client = None
+    try:
+        llm_client = LLMClient(cfg)
+    except ConfigError as e:
+        logging.warning("LLM config error: %s", e)
+    except Exception as e:
+        logging.warning("LLMClient init failed: %s", e)
+
+    # JSONL logger
+    try:
+        args.log_jsonl.parent.mkdir(parents=True, exist_ok=True)
+    except Exception:
+        pass
+    qalog = QALogger(args.log_jsonl)
+
+    # Index check
+    idx_ok, idx_msg = _index_ok_verbose()
+    if not idx_ok:
+        logging.info("Index check: %s", idx_msg)
+
+    # Memory (optional)
+    memory = None
+    mem_ok = False
+    if args.with_memory and MEMORY_AVAILABLE:
+        try:
+            memory = ConversationMemory("_out/memory/session.json")
+            mem_ok = True
+            logging.info("Conversation memory enabled")
+        except Exception as e:
+            logging.warning("Could not init memory: %s", e)
+
+    # Tool registration
+    extended_mode = args.extended_tools or detect_extended_features()
+    available_tools = openai_tools(extended=extended_mode)
+    logging.info("Registered %d tools (extended: %s)", len(available_tools), extended_mode)
+
+    # Domain terms for routing (if router supports it)
+    domain_terms = _extract_domain_terms(current)
+    if domain_terms:
+        # Try to inject into router (optional feature)
+        try:
+            from . import query_router
+            if hasattr(query_router, 'set_domain_terms'):
+                query_router.set_domain_terms(domain_terms)
+                query_router.set_fuzzy_threshold(0.84)
+                sample = ", ".join(list(sorted(domain_terms, key=len, reverse=True))[:5])
+                logging.info("Loaded %d domain terms (e.g., %s ...)", len(domain_terms), sample)
+            else:
+                logging.debug("Router does not support dynamic domain terms")
+        except Exception as e:
+            logging.debug("Could not inject domain terms into router: %s", e)
+
+    # Startup diagnostics
+    if args.verbose:
+        diag = _startup_diagnostics(
+            cfg,
+            llm_ok=bool(llm_client and llm_client.is_ready()),
+            idx_ok=idx_ok,
+            mem_ok=mem_ok
+        )
+        print(diag)
+
+    print("\n💬 Grant Analyst Chat ready! Type 'exit' to quit.\n")
+
+    # REPL loop
+    turn_count = 0
+    while True:
+        try:
+            user_input = input("You: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\n👋 Goodbye.")
+            break
+
+        if not user_input:
+            continue
+
+        if user_input.lower() in {"exit", "quit"}:
+            print("👋 Goodbye.")
+            break
+
+        turn_count += 1
+        t0 = time.time()
+
+        # Route intent
+        try:
+            routed = route(user_input, use_llm=args.use_llm_routing)
+        except Exception as e:
+            logging.warning("Routing failed (%s). Falling back to heuristic.", e)
+            routed = route(user_input, use_llm=False)
+
+        intent = str(routed.get("intent") or "general")
+        rargs = routed.get("args") or {}
+        answer_md = ""
+        ok = True
+
+        try:
+            # Handle intents with proper error handling
+            if intent in {"search", "list"}:
+                filt = (rargs.get("filters") or {}) if isinstance(rargs, dict) else {}
+                candidates = rargs.get("keyword_candidates") or []
+                primary_kw = rargs.get("keyword") or rargs.get("keyword_hint") or rargs.get("query") or ""
+                if primary_kw:
+                    candidates = [primary_kw] + [c for c in candidates if c != primary_kw]
+                if not candidates:
+                    candidates = [""]
+
+                res = []
+                used_kw = None
+                for kw in candidates:
+                    list_kwargs = {
+                        "keyword": (kw.strip() if isinstance(kw, str) else ""),
+                        "max_award": rargs.get("max_award") or filt.get("max_award"),
+                        "audience": rargs.get("audience") or filt.get("audience"),
+                        "status": rargs.get("status") or filt.get("status"),  # NEW: Add status filter
+                        "limit": rargs.get("limit"),  # FIXED: Don't default to 5, pass None for all
+                    }
+                    clean_kwargs = {k: v for k, v in list_kwargs.items() if v not in (None, "")}
+                    res = tools.list_grants(**clean_kwargs)
+                    if res:
+                        used_kw = kw
+                        break
+
+                if not res:
+                    answer_md = "No matching grants found."
+                else:
+                    hdr = f"### Results (matched on '{used_kw}')" if used_kw else "### Results"
+                    # Include status in display
+                    bullets = [
+                        f"- **{r['id']}** — {r['title']}\n"
+                        f"  Status: {r.get('status', 'unknown')} | Deadline: {r.get('deadline','n/a')}"
+                        for r in res
+                    ]
+                    # Add count summary
+                    status_counts = {}
+                    for r in res:
+                        s = r.get('status', 'unknown')
+                        status_counts[s] = status_counts.get(s, 0) + 1
+
+                    count_summary = f"\n**Found {len(res)} grant(s)**: " + \
+                        ", ".join(f"{count} {status}" for status, count in sorted(status_counts.items()))
+
+                    answer_md = hdr + count_summary + "\n\n" + "\n".join(bullets)
+
+            elif intent == "summarize":
+                row = tools.summarize_grant(rargs["grant_id"])
+                answer_md = row.get("summary_md", str(row))
+
+            elif intent == "compare":
+                diff = tools.compare_grants(rargs["grant_id_a"], rargs["grant_id_b"])
+                answer_md = diff.get("comparison_md", str(diff))
+
+            elif intent == "deadlines":
+                dl = tools.deadlines_overview(rargs.get("n", 5))
+                if not dl:
+                    answer_md = "No deadlines available."
+                else:
+                    answer_md = "### Upcoming deadlines\n" + "\n".join(
+                        f"- **{d['title']}** → {d['deadline']}" for d in dl
+                    )
+
+            else:
+                # General Q&A
+                if llm_client and llm_client.is_ready():
+                    answer_md = llm_client.summarize(user_input)
+                else:
+                    answer_md = "LLM not available. Try a structured command like `list battery` or `summarize competition-2316`."
+
+        except ValidationError as e:
+            ok = False
+            answer_md = f"⚠️  Invalid input: {e}"
+            logging.debug("Validation error: %s", e)
+
+        except DataLoadError as e:
+            ok = False
+            answer_md = f"⚠️  Data error: {e}"
+            logging.error("Data load error: %s", e)
+
+        except SearchError as e:
+            ok = False
+            answer_md = f"⚠️  Search error: {e}"
+            logging.error("Search error: %s", e)
+
+        except LLMError as e:
+            ok = False
+            answer_md = f"⚠️  LLM error: {e}\n💡 Tip: Check your API key and internet connection"
+            logging.error("LLM error: %s", e)
+
+        except GrantAnalyzerError as e:
+            ok = False
+            answer_md = f"⚠️  Error: {e}"
+            logging.error("Grant analyzer error: %s", e)
+
+        except Exception as e:
+            ok = False
+            answer_md = f"❌ Unexpected error: {e}"
+            logging.error("Unexpected error in chat turn", exc_info=True)
+
+        latency_ms = int((time.time() - t0) * 1000)
+        print(answer_md)
+
+        # Log turn
+        try:
+            qalog.write(
+                user=user_input, intent=intent, args=rargs,
+                answer_md=answer_md, ok=ok, latency_ms=latency_ms,
+                meta={
+                    "model": getattr(llm_client, "model", None),
+                    "provider": getattr(llm_client, "provider", None),
+                    "use_llm_routing": args.use_llm_routing,
+                    "extended_tools": extended_mode
+                }
+            )
+        except Exception:
+            pass
+
+        # Update memory (if enabled)
+        if memory:
+            try:
+                memory.add_turn("user", user_input)
+                memory.add_turn("assistant", answer_md)
+
+                # Periodic summarization (every 6 turns)
+                if turn_count % 6 == 0 and llm_client:
+                    memory.update_summary(llm_client.summarize)
+            except Exception as e:
+                logging.warning("Memory update failed: %s", e)
+
+
+if __name__ == "__main__":
+    main()

analyzer/chat/run_chat_llm.py

@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""
+run_chat_llm.py — Natural language chat with LLM-driven function calling
+
+This version uses the LLM's native function calling to handle ALL queries,
+making it much better at understanding complex natural language requests.
+
+Usage:
+    python -m src.analyzer.chat.run_chat_llm
+    python -m src.analyzer.chat.run_chat_llm --verbose
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from ..config import load_config
+from ..data_loader import load_current_grants, load_past_winners
+from ..llm_client import LLMClient
+from ..search.hybrid_index import load_index
+from ..utils.errors import (
+    GrantAnalyzerError,
+    ValidationError,
+    DataLoadError,
+    SearchError,
+    LLMError,
+    ConfigError
+)
+from .chat_tools import ChatTools
+from .tool_schemas import openai_tools, detect_extended_features
+
+
+def _startup_diagnostics(cfg: Dict, llm_ok: bool, idx_ok: bool) -> str:
+    """Generate startup diagnostics display."""
+    provider = cfg.get("llm_provider", "unknown")
+    model = cfg.get("llm_model", "unknown")
+    api_key = cfg.get("openai_api_key") or cfg.get("anthropic_api_key")
+
+    diag = [
+        "=== Grant Analyst Chat — LLM Function Calling Mode ===",
+        f"Provider:   {provider}",
+        f"Model:      {model}",
+        f"LLM Ready:  {llm_ok}",
+        f"Index OK:   {idx_ok}",
+        f"API Key:    {'✓' if api_key else '✗'}",
+        ""
+    ]
+    return "\n".join(diag)
+
+
+def _dispatch_tool_call(tools: ChatTools, tool_name: str, tool_args: Dict[str, Any]) -> Any:
+    """
+    Dispatch a tool call to the appropriate ChatTools method.
+
+    Args:
+        tools: ChatTools instance
+        tool_name: Name of the tool to call
+        tool_args: Arguments for the tool
+
+    Returns:
+        Tool result (dict or string)
+    """
+    try:
+        if tool_name == "list_grants":
+            return tools.list_grants(
+                keyword=tool_args.get("keyword"),
+                max_award=tool_args.get("max_award"),
+                audience=tool_args.get("audience"),
+                status=tool_args.get("status"),  # NEW: Add status filter
+                limit=tool_args.get("limit")  # FIXED: Don't default to 5, pass None for all
+            )
+
+        elif tool_name == "get_grant":
+            return tools.get_grant(tool_args["grant_id"])
+
+        elif tool_name == "summarize_grant":
+            return tools.summarize_grant(
+                tool_args["grant_id"],
+            )
+
+        elif tool_name == "compare_grants":
+            return tools.compare_grants(
+                tool_args["grant_id_a"],
+                tool_args["grant_id_b"]
+            )
+
+        elif tool_name == "deadlines_overview":
+            return tools.deadlines_overview(tool_args.get("n", 5))
+
+        elif tool_name == "analyze_company_for_grants":
+            return tools.analyze_company_for_grants(
+                tool_args["company_url"],
+                limit=tool_args.get("limit", 3)
+            )
+
+        elif tool_name == "search_grants":
+            # This would need to be implemented in ChatTools or InsightTools
+            return {
+                "results": tools.list_grants(
+                    keyword=tool_args.get("query"),
+                    status=tool_args.get("status"),  # NEW: Add status filter
+                    limit=tool_args.get("limit")  # FIXED: Don't default to 10, pass None for all
+                ),
+                "query": tool_args.get("query")
+            }
+
+        else:
+            return {"error": f"Unknown tool: {tool_name}"}
+
+    except Exception as e:
+        logging.error(f"Tool {tool_name} failed: {e}", exc_info=True)
+        return {"error": str(e)}
+
+
+def main(argv: Optional[List[str]] = None):
+    """Main chat loop with LLM function calling."""
+
+    # Parse arguments
+    ap = argparse.ArgumentParser(description="Grant Analyst Chat with LLM function calling")
+    ap.add_argument("--verbose", action="store_true", help="Show startup diagnostics")
+    ap.add_argument("--limit", type=int, help="Limit number of grants to load")
+    ap.add_argument("--extended-tools", action="store_true", help="Enable extended tools")
+    args = ap.parse_args(argv)
+
+    # Setup logging
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        handlers=[
+            logging.FileHandler("_out/logs/chat_llm.log"),
+            logging.StreamHandler(sys.stderr)
+        ]
+    )
+
+    # Load configuration
+    try:
+        cfg = load_config()
+    except ConfigError as e:
+        print(f"⚠️  Configuration error: {e}")
+        sys.exit(1)
+
+    # Load data
+    logging.info("Loading data...")
+    try:
+        current = load_current_grants(Path("data/snapshots"), limit=args.limit)
+        past = load_past_winners(
+            history_xlsx=Path("data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx")
+        )
+        logging.info(f"Loaded {len(current)} current grants; {len(past)} past winners")
+    except DataLoadError as e:
+        print(f"⚠️  Data loading error: {e}")
+        sys.exit(1)
+
+    # Initialize LLM client
+    try:
+        llm_client = LLMClient(cfg)
+        llm_ok = llm_client.is_ready()
+        if not llm_ok:
+            print("⚠️  LLM client not ready. Check your API key configuration.")
+            sys.exit(1)
+    except Exception as e:
+        print(f"⚠️  LLM initialization failed: {e}")
+        sys.exit(1)
+
+    # Load search index
+    idx_ok = False
+    try:
+        idx_path = Path("data/index/hybrid_index.pkl")
+        if idx_path.exists():
+            _ = load_index()
+            idx_ok = True
+    except Exception as e:
+        logging.warning(f"Could not load search index: {e}")
+
+    # Initialize ChatTools
+    tools = ChatTools(current, past)
+
+    # Tool registration
+    extended_mode = args.extended_tools or detect_extended_features()
+    available_tools = openai_tools(extended=extended_mode)
+    logging.info(f"Registered {len(available_tools)} tools (extended: {extended_mode})")
+
+    # Startup diagnostics
+    if args.verbose:
+        print(_startup_diagnostics(cfg, llm_ok, idx_ok))
+
+    print("\n💬 Grant Analyst Chat (LLM Function Calling Mode)")
+    print("   Ask me anything in natural language! Type 'exit' to quit.\n")
+
+    # Conversation history
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "You are a helpful grant analyst assistant for Innovate UK grants. "
+                "Use the available tools to answer user questions about grants, funding, "
+                "deadlines, eligibility, and comparisons. Always use tools when you need "
+                "to look up grant data - don't make up information. When listing grants, "
+                "format them clearly with bullets. When comparing, use tables. Be concise "
+                "but informative. Current date: 2025-10-22."
+            )
+        }
+    ]
+
+    # REPL loop
+    turn_count = 0
+    while True:
+        try:
+            user_input = input("You: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\n👋 Goodbye.")
+            break
+
+        if not user_input:
+            continue
+
+        if user_input.lower() in {"exit", "quit"}:
+            print("👋 Goodbye.")
+            break
+
+        turn_count += 1
+        t0 = time.time()
+
+        # Add user message
+        messages.append({"role": "user", "content": user_input})
+
+        try:
+            # Call LLM with function calling
+            response = llm_client.client.chat.completions.create(
+                model=llm_client.model,
+                messages=messages,
+                tools=available_tools,
+                tool_choice="auto",
+                temperature=0.1,
+            )
+
+            response_message = response.choices[0].message
+            tool_calls = response_message.tool_calls
+
+            # If LLM wants to call tools
+            if tool_calls:
+                # Add assistant's response with tool calls
+                messages.append(response_message)
+
+                # Execute each tool call
+                for tool_call in tool_calls:
+                    function_name = tool_call.function.name
+                    function_args = eval(tool_call.function.arguments)
+
+                    logging.info(f"Calling tool: {function_name} with args: {function_args}")
+
+                    # Execute the tool
+                    tool_result = _dispatch_tool_call(tools, function_name, function_args)
+
+                    # Add tool result to conversation
+                    messages.append({
+                        "role": "tool",
+                        "tool_call_id": tool_call.id,
+                        "name": function_name,
+                        "content": str(tool_result)
+                    })
+
+                # Get final response from LLM
+                final_response = llm_client.client.chat.completions.create(
+                    model=llm_client.model,
+                    messages=messages,
+                    temperature=0.1,
+                )
+
+                assistant_message = final_response.choices[0].message.content
+                messages.append({"role": "assistant", "content": assistant_message})
+            else:
+                # No tool calls, just use the response
+                assistant_message = response_message.content
+                messages.append({"role": "assistant", "content": assistant_message})
+
+            # Display response
+            print(f"\n{assistant_message}\n")
+
+            latency_ms = int((time.time() - t0) * 1000)
+            logging.info(f"Turn {turn_count} completed in {latency_ms}ms")
+
+        except LLMError as e:
+            print(f"\n⚠️  LLM error: {e}\n")
+            logging.error(f"LLM error: {e}")
+
+        except Exception as e:
+            print(f"\n❌ Error: {e}\n")
+            logging.error(f"Unexpected error: {e}", exc_info=True)
+
+
+if __name__ == "__main__":
+    main()

analyzer/chat/run_chat_plus.py

@@ -0,0 +1,39 @@
+# src/analyzer/chat/run_chat_plus.py
+"""
+DEPRECATED: This file has been merged into run_chat.py
+
+Migration:
+  python -m src.analyzer.chat.run_chat_plus --verbose
+  ↓
+  python -m src.analyzer.chat.run_chat --verbose
+
+All features are now available via flags in run_chat.py:
+  --verbose           Show startup diagnostics
+  --with-memory       Enable conversation memory
+  --extended-tools    Force extended tools
+  --no-llm-routing    Disable LLM routing
+  --limit N           Load at most N grants
+"""
+import warnings
+import sys
+
+warnings.warn(
+    "run_chat_plus.py is deprecated. Use run_chat.py with --verbose flag instead.",
+    DeprecationWarning,
+    stacklevel=2
+)
+
+# Redirect to new unified version
+from .run_chat import main
+
+if __name__ == "__main__":
+    print("⚠️  WARNING: run_chat_plus.py is deprecated")
+    print("   Use: python -m src.analyzer.chat.run_chat --verbose")
+    print("   Redirecting to new version...\n")
+
+    # Auto-enable verbose mode for backward compatibility
+    argv = sys.argv[1:]
+    if "--verbose" not in argv and "-v" not in argv:
+        argv = ["--verbose"] + argv
+
+    main(argv)

analyzer/chat/tool_schemas.py

@@ -0,0 +1,487 @@
+# src/analyzer/chat/tool_schemas.py
+"""
+Unified tool definitions for OpenAI + Anthropic with feature flags.
+
+Provides JSON schemas for all available tools. Use 'extended' flag to enable
+advanced features (insight_search, fetch_link) instead of maintaining separate files.
+"""
+
+from __future__ import annotations
+from typing import Any, Dict, List
+
+
+def _filters_schema() -> Dict[str, Any]:
+    """Shared filter schema for search operations."""
+    return {
+        "type": "object",
+        "properties": {
+            "status": {
+                "type": "string",
+                "description": "Grant status filter, e.g. 'open', 'closed', 'upcoming'.",
+                "enum": ["open", "closed", "upcoming"],
+            },
+            "audience": {
+                "type": "string",
+                "description": "Applicant type such as 'SME', 'university', 'research_organisation', etc."
+            },
+            "theme": {
+                "type": "string",
+                "description": "High-level theme or domain, e.g. 'battery', 'net zero', 'quantum', 'AI'."
+            },
+            "min_award": {"type": "number", "description": "Lower bound for funding (£)."},
+            "max_award": {"type": "number", "description": "Upper bound for funding (£)."},
+            "deadline_before": {
+                "type": "string",
+                "description": "Only include grants with a deadline before this ISO date (YYYY-MM-DD).",
+                "pattern": r"^\d{4}-\d{2}-\d{2}$"
+            },
+            "deadline_after": {
+                "type": "string",
+                "description": "Only include grants with a deadline on/after this ISO date (YYYY-MM-DD).",
+                "pattern": r"^\d{4}-\d{2}-\d{2}$"
+            },
+            "timeframe": {
+                "type": "string",
+                "description": "Loose time window like 'this_month', 'next_quarter'."
+            },
+        },
+        "additionalProperties": True,
+    }
+
+
+def openai_tools(*, extended: bool = False) -> List[Dict[str, Any]]:
+    """
+    Get tool schemas for OpenAI function calling.
+    
+    Args:
+        extended: If True, include advanced tools (insight_search, fetch_link)
+    
+    Returns:
+        List of OpenAI function call schemas
+    """
+    # Core tools (always included)
+    core_tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "search_grants",
+                "description": (
+                    "Natural-language search over grants with optional structured filters. "
+                    "Use this for messy user prompts or when you need fuzzy matching. "
+                    "If limit is not specified, returns ALL matching grants."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "User's search query in natural language."
+                        },
+                        "filters": _filters_schema(),
+                        "limit": {
+                            "type": "integer",
+                            "description": "Maximum results to return. If omitted, returns ALL matching grants."
+                        },
+                    },
+                    "required": ["query"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "list_grants",
+                "description": (
+                    "List all grants with optional filters by keyword, funding, status, or audience. "
+                    "Returns ALL matching grants if limit is not specified. "
+                    "Prefer 'search_grants' for natural language; use this for precise filtering."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "keyword": {
+                            "type": "string",
+                            "description": "Keyword to match in title or description."
+                        },
+                        "status": {
+                            "type": "string",
+                            "description": "Filter by grant status: 'open', 'closed', or 'upcoming'.",
+                            "enum": ["open", "closed", "upcoming"]
+                        },
+                        "max_award": {
+                            "type": "number",
+                            "description": "Upper funding bound (£)."
+                        },
+                        "audience": {
+                            "type": "string",
+                            "description": "Applicant type such as SME, university, etc."
+                        },
+                        "limit": {
+                            "type": "integer",
+                            "description": "Maximum results to return. If omitted, returns all matching grants.",
+                        },
+                    },
+                    "required": [],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_grant",
+                "description": "Retrieve full structured JSON for a grant by its ID.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "grant_id": {
+                            "type": "string",
+                            "description": "Grant ID (e.g., 'competition-2315' or '2315')"
+                        }
+                    },
+                    "required": ["grant_id"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "summarize_grant",
+                "description": "Generate an insightful markdown summary for a specific grant ID.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "grant_id": {
+                            "type": "string",
+                            "description": "Grant ID to summarize"
+                        },
+                        "audience": {
+                            "type": "string",
+                            "description": "Target audience style, e.g. 'exec', 'technical', 'sme', 'founder'.",
+                            "enum": ["exec", "technical", "sme", "founder"],
+                            "default": "exec"
+                        },
+                        "word_limit": {
+                            "type": "integer",
+                            "description": "Approximate word limit for summary",
+                            "default": 250
+                        }
+                    },
+                    "required": ["grant_id"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "summarize_grants_batch",
+                "description": (
+                    "Batch summarize multiple grants efficiently in parallel. "
+                    "Much faster than summarizing grants individually when you need summaries for multiple grants. "
+                    "Results stream back as they complete. Use this when user asks for 'summaries for all', "
+                    "'summarize X grants', or when processing multiple search results."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "grant_ids": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                            "description": "List of grant IDs to summarize (e.g., ['2313', '2314', '2315'])"
+                        },
+                        "batch_size": {
+                            "type": "integer",
+                            "description": "Number of grants to process per batch (default: 5)",
+                            "default": 5
+                        }
+                    },
+                    "required": ["grant_ids"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_all_grant_summaries",
+                "description": (
+                    "Get detailed summaries of ALL available grants in a single efficient batch operation. "
+                    "Perfect when user asks 'describe all grants', 'summaries of every grant', 'all grant opportunities', etc. "
+                    "Processes all grants in parallel batches for speed."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "batch_size": {
+                            "type": "integer",
+                            "description": "Number of grants to process per batch (default: 5)",
+                            "default": 5
+                        }
+                    },
+                    "required": [],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "compare_grants",
+                "description": (
+                    "Compare two grants side-by-side and highlight key differences in funding, "
+                    "deadlines, eligibility, and scope. Use this when user asks to 'compare' grants "
+                    "or wants to see differences between two grant IDs. Accepts IDs like '2313', "
+                    "'competition-2313', or 'grant-2313'."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "grant_id_a": {
+                            "type": "string",
+                            "description": "First grant ID (e.g., '2313', 'competition-2313')"
+                        },
+                        "grant_id_b": {
+                            "type": "string",
+                            "description": "Second grant ID (e.g., '2314', 'competition-2314')"
+                        },
+                    },
+                    "required": ["grant_id_a", "grant_id_b"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "deadlines_overview",
+                "description": "List upcoming deadlines for current grants, sorted by date.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "n": {
+                            "type": "integer",
+                            "description": "Number of deadlines to show",
+                            "default": 5
+                        }
+                    },
+                    "required": [],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "analyze_company_for_grants",
+                "description": (
+                    "Fetch and analyze a company website to recommend suitable grants. "
+                    "Takes a company URL, extracts information about what they do, and "
+                    "matches them with the most relevant available grants. Use this when "
+                    "the user provides a company website and asks 'what grants are suitable' "
+                    "or 'which grants should this company apply for'."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "company_url": {
+                            "type": "string",
+                            "description": "URL of the company website to analyze (e.g., 'https://example.com' or 'example.com')"
+                        },
+                        "limit": {
+                            "type": "integer",
+                            "description": "Number of grant recommendations to return",
+                            "default": 3
+                        }
+                    },
+                    "required": ["company_url"],
+                },
+            },
+        },
+    ]
+    
+    # Extended tools (only if requested)
+    extended_tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "insight_search",
+                "description": (
+                    "Answer a policy/eligibility question by searching the local index "
+                    "(grants + supporting docs). Returns grounded markdown with citations. "
+                    "Use this for 'how do I', 'what are the rules', 'am I eligible' questions."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "question": {
+                            "type": "string",
+                            "description": "User's question about grant policies, rules, or procedures"
+                        },
+                        "k": {
+                            "type": "integer",
+                            "description": "Number of supporting documents to retrieve",
+                            "default": 8
+                        },
+                    },
+                    "required": ["question"],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "fetch_link",
+                "description": (
+                    "Fetch and parse a URL (HTML or PDF) related to a grant. "
+                    "Returns plain text. Use this when the answer likely lives in "
+                    "a linked guidance/terms page not in the index."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "Full URL to fetch (must be https://)"
+                        },
+                        "force": {
+                            "type": "boolean",
+                            "description": "Force re-fetch even if cached",
+                            "default": False
+                        }
+                    },
+                    "required": ["url"],
+                },
+            },
+        },
+    ]
+    
+    return core_tools + (extended_tools if extended else [])
+
+
+def anthropic_tools(*, extended: bool = False) -> List[Dict[str, Any]]:
+    """
+    Get tool schemas for Anthropic function calling.
+    
+    Anthropic uses slightly different format (name at top level, input_schema
+    instead of parameters).
+    
+    Args:
+        extended: If True, include advanced tools
+    
+    Returns:
+        List of Anthropic function call schemas
+    """
+    openai_schemas = openai_tools(extended=extended)
+    
+    anthropic_schemas = []
+    for tool in openai_schemas:
+        fn = tool["function"]
+        anthropic_schemas.append({
+            "name": fn["name"],
+            "description": fn["description"],
+            "input_schema": fn["parameters"],
+        })
+    
+    return anthropic_schemas
+
+
+def get_tools(provider: str = "openai", *, extended: bool = False) -> List[Dict[str, Any]]:
+    """
+    Convenience function to get tools for any provider.
+    
+    Args:
+        provider: "openai" or "anthropic"
+        extended: Whether to include extended tools
+    
+    Returns:
+        Tool schemas appropriate for the provider
+    
+    Example:
+        tools = get_tools("openai", extended=True)
+        response = client.chat.completions.create(
+            model="gpt-5",
+            messages=[...],
+            tools=tools
+        )
+    """
+    provider = provider.lower()
+    if provider == "anthropic":
+        return anthropic_tools(extended=extended)
+    return openai_tools(extended=extended)
+
+
+# ---------------------------------------------------------------------------
+# Feature Detection
+# ---------------------------------------------------------------------------
+
+def detect_extended_features() -> bool:
+    """
+    Detect if extended features should be enabled based on environment.
+    
+    Checks for:
+    - ENABLE_EXTENDED_TOOLS=1
+    - Presence of hybrid index file
+    - Availability of fetcher module
+    
+    Returns:
+        True if extended tools should be enabled
+    """
+    import os
+    from pathlib import Path
+    
+    # Explicit override
+    if os.getenv("ENABLE_EXTENDED_TOOLS", "").lower() in ("1", "true", "yes"):
+        return True
+    
+    # Check if index exists (needed for insight_search)
+    index_path = Path("data/index/hybrid_index.pkl")
+    if not index_path.exists():
+        return False
+    
+    # Check if fetcher is available (needed for fetch_link)
+    try:
+        from ..net.fetcher import fetch_link
+        return True
+    except ImportError:
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Migration Helpers (Backward Compatibility)
+# ---------------------------------------------------------------------------
+
+def openai_tools_legacy() -> List[Dict[str, Any]]:
+    """
+    DEPRECATED: For backward compatibility with old code.
+    Use openai_tools(extended=False) instead.
+    """
+    import warnings
+    warnings.warn(
+        "openai_tools_legacy() is deprecated. Use openai_tools(extended=False).",
+        DeprecationWarning,
+        stacklevel=2
+    )
+    return openai_tools(extended=False)
+
+
+# ---------------------------------------------------------------------------
+# Self-Test
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import json
+    
+    print("=== Core Tools (OpenAI) ===")
+    core = openai_tools(extended=False)
+    print(f"Count: {len(core)}")
+    for tool in core:
+        print(f"  - {tool['function']['name']}")
+    
+    print("\n=== Extended Tools (OpenAI) ===")
+    extended = openai_tools(extended=True)
+    print(f"Count: {len(extended)}")
+    for tool in extended:
+        print(f"  - {tool['function']['name']}")
+    
+    print("\n=== Anthropic Format ===")
+    anthropic = anthropic_tools(extended=True)
+    print(f"Count: {len(anthropic)}")
+    print(json.dumps(anthropic[0], indent=2)[:300])
+    
+    print(f"\n=== Feature Detection ===")
+    print(f"Extended features available: {detect_extended_features()}")

analyzer/chat/tool_schemas_plus.py

@@ -0,0 +1,30 @@
+"""
+DEPRECATED: This module has been merged into tool_schemas.py
+
+Migration:
+  from .tool_schemas_plus import openai_tools
+  ↓
+  from .tool_schemas import openai_tools
+
+To enable extended features:
+  openai_tools(extended=True)
+  # or set ENABLE_EXTENDED_TOOLS=1 in environment
+"""
+import warnings
+warnings.warn(
+    "tool_schemas_plus is deprecated. Use tool_schemas with extended=True flag.",
+    DeprecationWarning,
+    stacklevel=2
+)
+
+# Thin wrapper for backward compatibility
+from .tool_schemas import openai_tools as _openai_tools
+from .tool_schemas import anthropic_tools as _anthropic_tools
+
+def openai_tools():
+    """DEPRECATED: Use tool_schemas.openai_tools(extended=True) instead."""
+    return _openai_tools(extended=True)
+
+def anthropic_tools():
+    """DEPRECATED: Use tool_schemas.anthropic_tools(extended=True) instead."""
+    return _anthropic_tools(extended=True)

analyzer/config.py

@@ -0,0 +1,120 @@
+"""
+config.py — central configuration for the summarizer MVP
+
+Minimal, dependency-free (std‑lib) config loader. Pulls settings from env vars
+with sensible defaults. Supports OpenAI *or* Anthropic via LLM_PROVIDER.
+
+ENV VARS (all optional unless noted)
+------------------------------------
+# Provider + model
+LLM_PROVIDER=openai|anthropic        # default: openai
+LLM_MODEL=                           # default per provider (see DEFAULT_MODELS)
+
+# Auth
+OPENAI_API_KEY=                      # required if LLM_PROVIDER=openai
+ANTHROPIC_API_KEY=                   # required if LLM_PROVIDER=anthropic
+
+# Behavior
+LLM_TEMPERATURE=0.2                  # 0..2
+LLM_MAX_OUTPUT_TOKENS=800            # model-dependent cap
+LLM_TIMEOUT_S=30                     # HTTP timeout (seconds)
+LLM_DISABLE=0                        # 1 disables external calls (draft-only)
+
+# Dev / tracing (optional)
+LOG_LEVEL=INFO                       # DEBUG|INFO|WARNING|ERROR
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Literal, Optional
+
+Provider = Literal["openai", "anthropic"]
+
+DEFAULT_MODELS = {
+    "openai": "gpt-5-mini",
+    "anthropic": "claude-3-5-haiku-20241022",  # fast, low-cost Claude
+}
+
+
+@dataclass
+class Config:
+    provider: Provider = "openai"
+    model: str = DEFAULT_MODELS["openai"]
+
+    openai_api_key: Optional[str] = None
+    anthropic_api_key: Optional[str] = None
+
+    # Model-specific configurations for different use cases
+    model_router: str = "gpt-5-nano"
+    model_translator: str = "gpt-5-mini"
+    model_analyzer: str = "gpt-5"
+
+    temperature: float = 0.2
+    max_output_tokens: int = 800
+    timeout_s: float = 30.0
+    disable_llm: bool = False
+
+    log_level: str = "INFO"
+
+    def validate(self) -> None:
+        p = self.provider
+        if p not in ("openai", "anthropic"):
+            raise ValueError(f"LLM_PROVIDER must be 'openai' or 'anthropic', got: {p}")
+        if not self.model:
+            raise ValueError("LLM_MODEL must be non-empty")
+        if self.disable_llm:
+            return  # no auth needed if disabled
+        if p == "openai" and not self.openai_api_key:
+            raise ValueError("OPENAI_API_KEY is required when LLM_PROVIDER=openai")
+        if p == "anthropic" and not self.anthropic_api_key:
+            raise ValueError("ANTHROPIC_API_KEY is required when LLM_PROVIDER=anthropic")
+        if not (0.0 <= float(self.temperature) <= 2.0):
+            raise ValueError("LLM_TEMPERATURE must be between 0 and 2")
+        if int(self.max_output_tokens) <= 0:
+            raise ValueError("LLM_MAX_OUTPUT_TOKENS must be > 0")
+        if float(self.timeout_s) <= 0:
+            raise ValueError("LLM_TIMEOUT_S must be > 0")
+
+
+def _env(name: str, default: Optional[str] = None) -> Optional[str]:
+    val = os.getenv(name)
+    return val if val is not None and val != "" else default
+
+
+def load_config() -> Config:
+    provider: Provider = cast_provider(_env("LLM_PROVIDER", "openai"))
+    model = _env("LLM_MODEL", DEFAULT_MODELS[provider])
+
+    cfg = Config(
+        provider=provider,
+        model=model,
+        openai_api_key=_env("OPENAI_API_KEY"),
+        anthropic_api_key=_env("ANTHROPIC_API_KEY"),
+        model_router=_env("LLM_MODEL_ROUTER", "gpt-5-nano"),
+        model_translator=_env("LLM_MODEL_TRANSLATOR", "gpt-5-mini"),
+        model_analyzer=_env("LLM_MODEL_ANALYZER", "gpt-5"),
+        temperature=float(_env("LLM_TEMPERATURE", "0.2")),
+        max_output_tokens=int(_env("LLM_MAX_OUTPUT_TOKENS", "800")),
+        timeout_s=float(_env("LLM_TIMEOUT_S", "30")),
+        disable_llm=_env("LLM_DISABLE", "0") == "1",
+        log_level=_env("LOG_LEVEL", "INFO"),
+    )
+    cfg.validate()
+    return cfg
+
+
+def cast_provider(s: Optional[str]) -> Provider:
+    s = (s or "openai").strip().lower()
+    if s in ("openai", "anthropic"):
+        return s  # type: ignore[return-value]
+    raise ValueError(f"Unsupported LLM_PROVIDER: {s}")
+
+
+# Quick self-test when run directly
+if __name__ == "__main__":
+    try:
+        c = load_config()
+        print("Config loaded:\n", c)
+    except Exception as e:
+        print("Config error:", e)

analyzer/context_builder.py

@@ -0,0 +1,274 @@
+"""
+context_builder.py — prepares text context for the LLM summarizer
+
+Takes structured JSON grant records + optional past winners and builds a single
+text block for each grant, suitable as LLM input.
+
+Responsibilities:
+- Extract key text fields from each grant JSON (title, description, sections)
+- Summarize/flatten them into a readable context string
+- Optionally include supporting documents (HTML sections + PDF text extracts)
+- Optionally include a few relevant past-winner snippets (if any exist)
+
+Public API
+----------
+build_context(grant: dict, past_winners: list[dict] | None = None, include_supporting: bool = False) -> str
+build_context_with_supporting(grant: dict, k: int = 5, past_winners: list[dict] | None = None) -> str
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional, Tuple
+import re
+import json
+from pathlib import Path
+
+# ----------------------------- Text utilities ---------------------------------
+
+def _clean(s: Any) -> str:
+    return re.sub(r"\s+", " ", str(s or "")).strip()
+
+
+def _maybe(k: str, v: Any) -> str:
+    if not v:
+        return ""
+    return f"{k}: {_clean(v)}\n"
+
+# ----------------------------- Context builder --------------------------------
+
+def build_context(grant: Dict[str, Any], past_winners: Optional[List[Dict[str, Any]]] = None) -> str:
+    """
+    Flatten a grant JSON object + optional past winners into a readable context string.
+
+    CRITICAL: All URLs are provided in FULL form (https://...), never partial paths.
+    The model receives complete, actionable URLs directly from the snapshot JSON.
+    """
+    lines: List[str] = []
+
+    # --- Basic info ---
+    title = grant.get("title") or grant.get("name") or grant.get("competition_title")
+    if title:
+        lines.append(f"TITLE: {_clean(title)}")
+
+    # --- FULL URL: Always provide complete https:// URLs, never relative paths ---
+    url = grant.get("url") or grant.get("link")
+    if url:
+        lines.append(f"URL: {url}")  # Example: "https://apply-for-innovation-funding.service.gov.uk/competition/2276/overview/..."
+
+    # --- Funding information ---
+    funding = grant.get("funding") or {}
+    if isinstance(funding, dict):
+        max_funding = funding.get("max")
+        if max_funding:
+            lines.append(f"FUNDING: Up to £{max_funding:,}")
+        min_funding = funding.get("min")
+        if min_funding and min_funding != max_funding:
+            lines.append(f"MINIMUM FUNDING: £{min_funding:,}")
+    else:
+        funding_text = grant.get("funding_amount") or grant.get("amount") or grant.get("funding_rate")
+        if funding_text:
+            lines.append(f"FUNDING: {_clean(funding_text)}")
+
+    # --- Dates ---
+    deadline = grant.get("close_date") or grant.get("deadline")
+    if deadline:
+        lines.append(f"DEADLINE: {_clean(deadline)}")
+
+    open_date = grant.get("open_date")
+    if open_date:
+        lines.append(f"OPENS: {_clean(open_date)}")
+
+    # --- Core text sections from snapshot ---
+    sections = grant.get("sections") or {}
+    if sections:
+        # Process snapshot sections in order: summary, eligibility, scope, dates, how_to_apply
+        section_order = [
+            "summary_raw", "eligibility_raw", "scope_raw",
+            "dates_raw", "how_to_apply_raw", "supporting_information_raw"
+        ]
+        for section_key in section_order:
+            v = sections.get(section_key)
+            if v and _clean(v):  # Only include non-empty sections
+                section_name = section_key.replace("_raw", "").replace("_", " ").upper()
+                lines.append(f"\n{section_name}:\n{_clean(v)}")
+    else:
+        # Fallback to common text fields
+        desc = grant.get("description") or grant.get("summary") or grant.get("scope")
+        if desc:
+            lines.append(f"\nDESCRIPTION:\n{_clean(desc)}")
+
+    # --- Optional extras ---
+    eligibility = grant.get("eligibility")
+    if eligibility:
+        lines.append(f"\nELIGIBILITY:\n{_clean(eligibility)}")
+
+    scope = grant.get("scope")
+    if scope:
+        lines.append(f"\nSCOPE:\n{_clean(scope)}")
+
+    # --- Contact Information ---
+    lines.append("\n--- CONTACT INFORMATION ---")
+    lines.append("Email: support@iuk.ukri.org")
+    lines.append("Phone: 0300 321 4357")
+    lines.append("Hours: 9am-12pm, 2pm-5pm, Monday-Friday (excluding bank holidays)")
+
+    # --- Past winners summary ---
+    if past_winners:
+        lines.append("\n--- RELATED PAST WINNERS ---")
+        for w in past_winners[:5]:  # limit to top 5 to avoid overloading tokens
+            snippet_parts: List[str] = []
+            snippet_parts.append(_maybe("Project", w.get("project_title")))
+            snippet_parts.append(_maybe("Organisation", w.get("lead_org")))
+            snippet_parts.append(_maybe("Award", w.get("award_amount")))
+            snippet_parts.append(_maybe("Competition", w.get("competition")))
+            abs_ = _clean(w.get("abstract"))
+            if abs_:
+                snippet_parts.append(f"Abstract: {abs_[:400]}{'…' if len(abs_)>400 else ''}\n")
+            lines.append("".join(snippet_parts))
+
+    # --- Return ---
+    context_text = "\n".join(lines).strip()
+    return context_text
+
+
+# ----------------------------- Supporting documents loader --------------------------------
+
+def _load_supporting_docs_jsonl(path: Optional[str] = None) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Load the supporting documents JSONL file and index by grant_id.
+
+    Returns: dict[grant_id] -> list of supporting docs
+    """
+    if path is None:
+        path = "data/supporting_jsonl/docs.jsonl"
+
+    try:
+        p = Path(path)
+        if not p.exists():
+            return {}
+
+        indexed = {}
+        with open(p, "r", encoding="utf-8") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                try:
+                    doc = json.loads(line)
+                    gid = doc.get("grant_id", "").replace("competition-", "")
+                    if gid:
+                        if gid not in indexed:
+                            indexed[gid] = []
+                        indexed[gid].append(doc)
+                except json.JSONDecodeError:
+                    continue
+        return indexed
+    except Exception:
+        return {}
+
+
+def get_supporting_docs_for_grant(grant_id: str, k: int = 5, doc_types: Optional[List[str]] = None) -> List[Dict[str, str]]:
+    """
+    Retrieve supporting documents for a grant.
+
+    Args:
+        grant_id: Grant ID (with or without "competition-" prefix)
+        k: Number of documents to return
+        doc_types: Filter by document type ("supporting_html", "supporting_pdf", etc.)
+
+    Returns: List of documents with extracted text
+    """
+    # Load cache on first use (could be cached module-level)
+    cache = _load_supporting_docs_jsonl()
+
+    # Normalize grant ID
+    gid = str(grant_id).replace("competition-", "").strip()
+    docs = cache.get(gid, [])
+
+    # Filter by type if requested
+    if doc_types:
+        docs = [d for d in docs if d.get("doc_type") in doc_types]
+
+    # Return top k
+    return docs[:k]
+
+
+def build_context_with_supporting(
+    grant: Dict[str, Any],
+    k: int = 5,
+    pdf_only: bool = False,
+    past_winners: Optional[List[Dict[str, Any]]] = None
+) -> str:
+    """
+    Build context including supporting documents (PDFs + HTML sections).
+
+    Args:
+        grant: Grant dict with id/competition_id field (or will extract from URL)
+        k: Number of supporting docs to include
+        pdf_only: If True, only include PDF documents
+        past_winners: Optional past winners for comparison
+
+    Returns: Context string with supporting materials embedded
+    """
+    # Start with base context
+    lines = [build_context(grant, past_winners)]
+
+    # Extract grant ID from multiple sources
+    gid = grant.get("id") or grant.get("competition_id")
+
+    # If not found, try extracting from URL
+    if not gid:
+        url = grant.get("url") or ""
+        match = re.search(r"/competition/(\d+)", url)
+        if match:
+            gid = match.group(1)
+
+    if not gid:
+        return lines[0]
+
+    # Filter document types
+    doc_types = ["supporting_pdf"] if pdf_only else ["supporting_pdf", "supporting_html"]
+
+    # Load supporting docs
+    supporting = get_supporting_docs_for_grant(str(gid), k=k, doc_types=doc_types)
+
+    if supporting:
+        lines.append("\n" + "="*80)
+        lines.append("SUPPORTING MATERIALS & PDF CONTENT:")
+        lines.append("="*80)
+
+        for i, doc in enumerate(supporting, 1):
+            doc_type = doc.get("doc_type", "unknown")
+            section = doc.get("section", "Supporting Info")
+            text = doc.get("text", "")
+
+            # Truncate long text but keep it substantial
+            if len(text) > 2000:
+                text = text[:2000] + "\n[... truncated ...]"
+
+            lines.append(f"\n[{i}] {section.upper()} ({doc_type})")
+            lines.append("-" * 60)
+            lines.append(text)
+
+    return "\n".join(lines)
+
+
+# Self-test
+if __name__ == "__main__":
+    fake_grant = {
+        "title": "AI Battery Research Program",
+        "funding_amount": "up to £1M",
+        "deadline": "2025-12-17",
+        "sections": {
+            "summary_raw": "Funding for early-stage AI-driven battery optimization.",
+            "scope_raw": "Projects must demonstrate significant improvement in energy density.",
+        },
+    }
+    fake_winners = [
+        {
+            "project_title": "BatteryX AI",
+            "lead_org": "EnergyAI Ltd",
+            "award_amount": "£500,000",
+            "competition": "Battery Innovation 2023",
+            "abstract": "Developed machine learning models for lithium-ion battery efficiency.",
+        }
+    ]
+    print(build_context(fake_grant, fake_winners))

analyzer/crawler/discover_grants.py

@@ -0,0 +1,306 @@
+"""
+Discover and fetch new grants from Innovate UK website.
+
+This module:
+1. Fetches the Innovate UK competition search/listing page
+2. Extracts all available grant URLs
+3. Fetches each grant's details using snapshot.py
+4. Saves to snapshots directory
+"""
+
+import logging
+import asyncio
+import json
+import re
+from pathlib import Path
+from typing import List, Tuple, Optional, Set
+from urllib.parse import urljoin, urlparse
+from datetime import datetime, UTC
+from playwright.async_api import async_playwright
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+# Innovate UK service base URL
+IUK_BASE = "https://apply-for-innovation-funding.service.gov.uk"
+COMPETITIONS_URL = f"{IUK_BASE}/competition/search"
+
+
+async def discover_grant_urls(max_retries: int = 2) -> List[str]:
+    """
+    Discover all available grant overview URLs from Innovate UK.
+
+    Returns:
+        List of grant overview URLs
+    """
+    logger.info(f"Discovering grants from {COMPETITIONS_URL}")
+
+    async with async_playwright() as pw:
+        browser = await pw.chromium.launch(
+            headless=True,
+            args=["--disable-dev-shm-usage"]
+        )
+        context = await browser.new_context(
+            user_agent=(
+                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+            ),
+            locale="en-GB",
+            timezone_id="Europe/London",
+        )
+        page = await context.new_page()
+        page.set_default_timeout(30000)
+
+        for attempt in range(max_retries):
+            try:
+                await page.goto(COMPETITIONS_URL, wait_until="domcontentloaded")
+                await page.wait_for_load_state("networkidle")
+                break
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    logger.error(f"Failed to fetch competitions page: {e}")
+                    await context.close()
+                    await browser.close()
+                    return []
+                logger.warning(f"Attempt {attempt + 1} failed, retrying...")
+
+        html = await page.content()
+        await context.close()
+        await browser.close()
+
+    # Parse URLs from HTML
+    urls = _extract_grant_urls(html)
+    logger.info(f"Discovered {len(urls)} grants")
+
+    return urls
+
+
+def _extract_grant_urls(html: str) -> List[str]:
+    """
+    Extract all grant overview URLs from the competitions listing page.
+
+    Looks for links matching pattern: /competition/{id}/overview/{uuid}
+    """
+    soup = BeautifulSoup(html, "lxml")
+    urls = []
+
+    # Find all links that match the overview pattern
+    pattern = re.compile(r'/competition/(\d+)/overview/([0-9a-f\-]{8,})', re.I)
+
+    for link in soup.find_all("a", href=True):
+        href = link.get("href", "")
+        if pattern.search(href):
+            # Make absolute URL
+            full_url = urljoin(IUK_BASE, href)
+            if full_url not in urls:
+                urls.append(full_url)
+
+    # Also check for links in data attributes or javascript
+    for elem in soup.find_all(["a", "div", "li"], {"data-href": True}):
+        href = elem.get("data-href", "")
+        if pattern.search(href):
+            full_url = urljoin(IUK_BASE, href)
+            if full_url not in urls:
+                urls.append(full_url)
+
+    return urls
+
+
+async def fetch_grant_snapshot(url: str, output_dir: Path) -> Optional[str]:
+    """
+    Fetch a single grant's snapshot and save to JSON.
+
+    Args:
+        url: Grant overview URL
+        output_dir: Directory to save snapshot JSON
+
+    Returns:
+        Filename of saved snapshot, or None if failed
+    """
+    try:
+        from .snapshot import fetch_sections_from_overview, parse_deeplink
+
+        # Extract grant ID from URL
+        match = re.search(r'/competition/(\d+)/', url)
+        if not match:
+            logger.warning(f"Could not extract grant ID from {url}")
+            return None
+
+        grant_id = match.group(1)
+        output_path = output_dir / f"competition-{grant_id}.json"
+
+        # Skip if already exists
+        if output_path.exists():
+            logger.debug(f"Grant {grant_id} already exists, skipping")
+            return None
+
+        logger.info(f"Fetching grant {grant_id}...")
+        html, sections = await fetch_sections_from_overview(url)
+
+        # Parse dates and funding from the fetched content
+        from .snapshot import (
+            parse_dates_singleline,
+            extract_funding,
+            _find_duration_months,
+            pick_open_close_from_milestones,
+            derive_aux_dates,
+            clean_title
+        )
+
+        # Parse dates
+        dates_text = sections.get("dates_raw", "") or ""
+        milestones = parse_dates_singleline(dates_text)
+
+        # Derive open/close from milestones
+        open_date, close_date = pick_open_close_from_milestones(milestones)
+
+        # If either missing, try scanning all text
+        if not open_date or not close_date:
+            all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
+            extra = parse_dates_singleline(all_text)
+            seen = {(m["label_raw"], m["date_iso"], m.get("date_iso_end")) for m in milestones}
+            for m2 in extra:
+                key = (m2["label_raw"], m2["date_iso"], m2.get("date_iso_end"))
+                if key not in seen:
+                    milestones.append(m2)
+                    seen.add(key)
+            od2, cd2 = pick_open_close_from_milestones(milestones)
+            open_date = open_date or od2
+            close_date = close_date or cd2
+
+        notify_date, project_start_from = derive_aux_dates(milestones)
+
+        # Parse funding
+        funding = extract_funding(sections)
+
+        # Parse duration
+        dur_min, dur_max = _find_duration_months(sections.get("eligibility_raw", "") or "")
+        if dur_min is None or dur_max is None:
+            all_text_for_duration = " ".join(v for v in sections.values() if isinstance(v, str) and v)
+            dur_min, dur_max = _find_duration_months(all_text_for_duration)
+        duration_months = {"min": dur_min, "max": dur_max}
+
+        # Extract title
+        raw_title = sections.get("summary_raw", "").split("\n")[0] if sections.get("summary_raw") else ""
+        title = clean_title(raw_title)
+
+        # Create snapshot
+        snapshot = {
+            "id": f"competition-{grant_id}",
+            "competition_id": grant_id,
+            "url": url,
+            "title": title,
+            "programme": "",
+            "round": "",
+            "open_date": open_date,
+            "close_date": close_date,
+            "notify_date": notify_date,
+            "project_start_from": project_start_from,
+            "funding": funding,
+            "duration_months": duration_months,
+            "sections": sections,
+            "pdfs": [],
+            "summaries": {},
+            "extracted": {"milestones": milestones},
+            "wonky": {"score": 0.0, "reasons": []},
+            "prev_round_refs": [],
+            "diff_summary": "",
+            "history_stats": {},
+            "created_at": datetime.now(UTC).isoformat(),
+            "updated_at": datetime.now(UTC).isoformat(),
+        }
+
+        # Save snapshot
+        output_path.write_text(
+            json.dumps(snapshot, indent=2),
+            encoding="utf-8"
+        )
+        logger.info(f"Saved grant {grant_id} snapshot")
+
+        return output_path.name
+
+    except Exception as e:
+        logger.error(f"Failed to fetch grant from {url}: {e}")
+        return None
+
+
+async def discover_and_fetch_grants(
+    output_dir: Path,
+    skip_existing: bool = True
+) -> Tuple[int, int, List[str]]:
+    """
+    Discover all grants and fetch new ones.
+
+    Args:
+        output_dir: Directory to save snapshots
+        skip_existing: Skip grants that already exist
+
+    Returns:
+        Tuple of (total_discovered, newly_fetched, new_filenames)
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Step 1: Discover all grant URLs
+    urls = await discover_grant_urls()
+    if not urls:
+        logger.warning("No grants discovered")
+        return 0, 0, []
+
+    # Step 2: Get existing grant IDs if skipping
+    existing_ids: Set[str] = set()
+    if skip_existing:
+        for json_file in output_dir.glob("competition-*.json"):
+            match = re.search(r'competition-(\d+)', json_file.name)
+            if match:
+                existing_ids.add(match.group(1))
+
+    # Step 3: Fetch new grants concurrently
+    new_files = []
+    tasks = []
+
+    for url in urls:
+        match = re.search(r'/competition/(\d+)/', url)
+        if match and match.group(1) in existing_ids:
+            logger.debug(f"Grant {match.group(1)} already exists")
+            continue
+
+        tasks.append(fetch_grant_snapshot(url, output_dir))
+
+    if tasks:
+        logger.info(f"Fetching {len(tasks)} new grants concurrently...")
+        results = await asyncio.gather(*tasks, return_exceptions=False)
+        new_files = [f for f in results if f is not None]
+
+    logger.info(
+        f"Discovery complete: {len(urls)} total, "
+        f"{len(new_files)} newly fetched"
+    )
+
+    return len(urls), len(new_files), new_files
+
+
+def main_sync(
+    output_dir: str = "data/snapshots",
+    skip_existing: bool = True
+) -> Tuple[int, int, List[str]]:
+    """
+    Synchronous wrapper for discovering and fetching grants.
+    """
+    output_path = Path(output_dir)
+    return asyncio.run(
+        discover_and_fetch_grants(output_path, skip_existing)
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+
+    total, new, files = main_sync()
+    print(f"\n✓ Discovery complete:")
+    print(f"  Total discovered: {total}")
+    print(f"  Newly fetched: {new}")
+    if files:
+        print(f"  Files: {', '.join(files)}")

analyzer/crawler/scheduler.py

@@ -0,0 +1,514 @@
+"""
+Automated Grant Scraping Scheduler
+
+Runs periodic crawls of Innovate UK and other funding sources to:
+- Fetch new grants and update the database
+- Mark grants as closed when deadlines pass
+- Deduplicate entries
+- Refresh the search index
+
+Configuration:
+- Run daily at 2 AM (configurable via CRAWLER_HOUR)
+- Supports both APScheduler and system cron integration
+"""
+
+import logging
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, Dict, Any
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+# Configuration
+DEFAULT_CRAWL_HOUR = int(os.getenv("CRAWLER_HOUR", "2"))  # 2 AM
+CRAWL_ENABLED = os.getenv("CRAWL_ENABLED", "false").lower() in ("true", "1", "yes")
+SNAPSHOTS_DIR = Path(os.getenv("SNAPSHOTS_DIR", "data/snapshots"))
+INDEX_PATH = Path(os.getenv("INDEX_PATH", "data/index/hybrid_index.pkl"))
+
+
+@dataclass
+class CrawlResult:
+    """Result of a crawl operation."""
+    timestamp: datetime
+    new_grants: int
+    updated_grants: int
+    closed_grants: int
+    duplicates_removed: int
+    index_rebuilt: bool
+    error: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "timestamp": self.timestamp.isoformat(),
+            "new_grants": self.new_grants,
+            "updated_grants": self.updated_grants,
+            "closed_grants": self.closed_grants,
+            "duplicates_removed": self.duplicates_removed,
+            "index_rebuilt": self.index_rebuilt,
+            "error": self.error,
+        }
+
+
+def deduplicate_grants(snapshots_dir: Path) -> int:
+    """
+    Check for duplicate grants and keep only the most recent.
+
+    Returns:
+        Number of duplicates removed
+    """
+    import json
+
+    if not snapshots_dir.exists():
+        return 0
+
+    # Group grants by ID (handles competition-XXXX variants)
+    grants_by_id: Dict[str, list] = {}
+
+    for json_file in snapshots_dir.glob("*.json"):
+        try:
+            with open(json_file, "r", encoding="utf-8") as f:
+                grant = json.load(f)
+
+            grant_id = grant.get("id") or json_file.stem
+            if grant_id not in grants_by_id:
+                grants_by_id[grant_id] = []
+
+            grants_by_id[grant_id].append({
+                "file": json_file,
+                "timestamp": json_file.stat().st_mtime,
+                "data": grant
+            })
+        except Exception as e:
+            logger.warning(f"Failed to read {json_file}: {e}")
+            continue
+
+    # Remove older duplicates
+    removed = 0
+    for grant_id, versions in grants_by_id.items():
+        if len(versions) > 1:
+            # Sort by timestamp, keep newest
+            versions.sort(key=lambda x: x["timestamp"], reverse=True)
+
+            for old_version in versions[1:]:
+                try:
+                    old_version["file"].unlink()
+                    removed += 1
+                    logger.info(f"Removed duplicate: {old_version['file'].name}")
+                except Exception as e:
+                    logger.warning(f"Failed to remove {old_version['file']}: {e}")
+
+    return removed
+
+
+def mark_closed_grants(snapshots_dir: Path) -> int:
+    """
+    Scan grants and mark those with passed deadlines as 'closed'.
+
+    Returns:
+        Number of grants marked as closed
+    """
+    import json
+    from datetime import datetime
+
+    if not snapshots_dir.exists():
+        return 0
+
+    now = datetime.now()
+    closed_count = 0
+
+    for json_file in snapshots_dir.glob("*.json"):
+        try:
+            with open(json_file, "r", encoding="utf-8") as f:
+                grant = json.load(f)
+
+            # Check deadline
+            deadline_str = grant.get("close_date") or grant.get("deadline")
+            if not deadline_str:
+                continue
+
+            # Parse deadline
+            try:
+                # Handle ISO format with time
+                if "T" in deadline_str:
+                    deadline = datetime.fromisoformat(deadline_str.replace("Z", "+00:00"))
+                else:
+                    deadline = datetime.fromisoformat(deadline_str)
+            except ValueError:
+                continue
+
+            # If deadline passed and not marked closed, update it
+            if deadline < now and grant.get("status") != "closed":
+                grant["status"] = "closed"
+                grant["marked_closed_at"] = now.isoformat()
+
+                with open(json_file, "w", encoding="utf-8") as f:
+                    json.dump(grant, f, indent=2)
+
+                logger.info(f"Marked as closed: {grant.get('title', json_file.stem)}")
+                closed_count += 1
+
+        except Exception as e:
+            logger.warning(f"Failed to process {json_file}: {e}")
+            continue
+
+    return closed_count
+
+
+def rebuild_search_index(
+    snapshots_dir: Path = SNAPSHOTS_DIR,
+    output_path: Path = INDEX_PATH
+) -> bool:
+    """
+    Rebuild the search index from current data.
+
+    Returns:
+        True if successful
+    """
+    try:
+        from ..search.hybrid_index import rebuild_index_from_data
+        from ..search.past_winners_integration import (
+            enrich_index_with_past_winners
+        )
+        from ..data_loader import load_past_winners
+
+        logger.info("Rebuilding search index...")
+
+        # Rebuild main index
+        idx = rebuild_index_from_data(
+            snapshots_dir=str(snapshots_dir),
+            output_path=str(output_path)
+        )
+
+        # Try to enrich with past winners
+        try:
+            past_winners = load_past_winners()
+            if past_winners:
+                enrich_index_with_past_winners(idx, past_winners)
+                logger.info(f"Enhanced index with {len(past_winners)} past winners")
+        except Exception as e:
+            logger.warning(f"Could not integrate past winners: {e}")
+
+        logger.info("Search index rebuilt successfully")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to rebuild search index: {e}")
+        return False
+
+
+def generate_summaries_for_grants(
+    snapshots_dir: Path,
+    grant_ids: Optional[list] = None
+) -> int:
+    """
+    Generate layman, technical, and executive summaries for grants.
+
+    Args:
+        snapshots_dir: Directory containing grant JSON files
+        grant_ids: Optional list of grant IDs to process (defaults to all)
+
+    Returns:
+        Number of grants summarized
+    """
+    import json
+    import asyncio
+
+    try:
+        from ...database import SummaryStore
+        from ...analyzer.config import load_config
+        from ...analyzer.llm_client import LLMClient
+        from ...analyzer.summarizer_optimized import extract_minimal_context
+    except ImportError as e:
+        logger.error(f"Required modules not available: {e}")
+        return 0
+
+    if not snapshots_dir.exists():
+        return 0
+
+    # Initialize MongoDB and LLM
+    try:
+        summary_store = SummaryStore()
+        config = load_config()
+        # Override to use gpt-5-mini for overnight batch processing
+        config.model = "gpt-5-mini"
+        llm_client = LLMClient(config)
+    except Exception as e:
+        logger.error(f"Failed to initialize summary generation: {e}")
+        return 0
+
+    # Load grants to summarize
+    grants_to_process = []
+    for json_file in snapshots_dir.glob("*.json"):
+        try:
+            with open(json_file, "r", encoding="utf-8") as f:
+                grant = json.load(f)
+
+            grant_id = grant.get("id") or json_file.stem
+
+            # Filter by grant_ids if provided
+            if grant_ids and grant_id not in grant_ids:
+                continue
+
+            grants_to_process.append(grant)
+        except Exception as e:
+            logger.warning(f"Failed to read {json_file}: {e}")
+
+    if not grants_to_process:
+        logger.info("No grants to summarize")
+        return 0
+
+    logger.info(f"Generating summaries for {len(grants_to_process)} grants...")
+
+    # Generate summaries in parallel batches with bulk write
+    async def generate_all_summaries():
+        tasks = []
+        for grant in grants_to_process:
+            tasks.append(generate_grant_summaries(grant, llm_client))
+
+        # Process in parallel (batches of 10)
+        all_summaries = []
+        for i in range(0, len(tasks), 10):
+            batch = tasks[i:i+10]
+            batch_results = await asyncio.gather(*batch, return_exceptions=True)
+
+            # Collect all successful summaries for bulk write
+            for result in batch_results:
+                if isinstance(result, list):
+                    all_summaries.extend(result)
+
+        # Bulk write all summaries at once (more efficient)
+        if all_summaries:
+            saved_count = summary_store.bulk_save_summaries(all_summaries)
+            logger.info(f"Bulk saved {saved_count} summaries")
+            return saved_count // 3  # Divide by 3 since we generate 3 types per grant
+
+        return 0
+
+    async def generate_grant_summaries(grant, client):
+        """Generate all 3 summary types for a single grant (returns list of summaries)."""
+        grant_id = grant.get("id", "unknown")
+
+        try:
+            # Extract minimal context
+            context = extract_minimal_context(grant)
+
+            # Generate 3 summary types in parallel
+            summary_types = [
+                ("layman", "Explain this grant in simple, everyday language that anyone can understand."),
+                ("technical", "Provide a detailed technical summary of this grant, including eligibility criteria and funding details."),
+                ("exec", "Provide a concise executive summary highlighting key points and deadlines.")
+            ]
+
+            async def generate_typed_summary(summary_type, instruction):
+                prompt = f"{instruction}\n\n{context}"
+
+                try:
+                    # Use streaming=False for batch processing
+                    summary = client.summarize(prompt, max_tokens=300)
+                    return {
+                        "grant_id": grant_id,
+                        "summary_type": summary_type,
+                        "summary_text": summary,
+                        "metadata": {"model": client.model, "context_length": len(context)}
+                    }
+                except Exception as e:
+                    logger.error(f"Failed to generate {summary_type} summary for {grant_id}: {e}")
+                    return None
+
+            # Generate all 3 types in parallel
+            results = await asyncio.gather(*[
+                generate_typed_summary(stype, instruction)
+                for stype, instruction in summary_types
+            ], return_exceptions=True)
+
+            # Filter out None results
+            return [r for r in results if r is not None and isinstance(r, dict)]
+
+        except Exception as e:
+            logger.error(f"Failed to process grant {grant_id}: {e}")
+            return []
+
+    try:
+        summarized_count = asyncio.run(generate_all_summaries())
+        logger.info(f"Successfully generated summaries for {summarized_count} grants")
+        return summarized_count
+    except Exception as e:
+        logger.error(f"Summary generation failed: {e}", exc_info=True)
+        return 0
+
+
+def run_crawl_cycle() -> CrawlResult:
+    """
+    Run a complete crawl and maintenance cycle.
+
+    This is the main entry point for scheduled crawls.
+    """
+    import asyncio
+
+    logger.info("Starting grant crawler cycle...")
+    start_time = datetime.now()
+
+    try:
+        # Step 1: Discover and fetch new grants from Innovate UK
+        new_grants = 0
+        new_grant_ids = []
+        try:
+            from ...crawler.discover_grants import discover_and_fetch_grants
+
+            logger.info("Discovering new grants from Innovate UK...")
+            total_discovered, newly_fetched, new_files = asyncio.run(
+                discover_and_fetch_grants(SNAPSHOTS_DIR, skip_existing=True)
+            )
+            new_grants = newly_fetched
+            new_grant_ids = [f.replace(".json", "") for f in new_files]
+            logger.info(
+                f"Grant discovery: {total_discovered} total, "
+                f"{newly_fetched} newly fetched"
+            )
+            if new_files:
+                logger.info(f"New grants: {', '.join(new_files)}")
+        except Exception as e:
+            logger.error(f"Grant discovery failed: {e}", exc_info=True)
+            # Continue with other steps even if discovery fails
+
+        # Step 2: Generate summaries for new grants (using gpt-5-mini)
+        summaries_generated = 0
+        if new_grant_ids:
+            try:
+                logger.info(f"Generating layman summaries for {len(new_grant_ids)} new grants...")
+                summaries_generated = generate_summaries_for_grants(
+                    SNAPSHOTS_DIR,
+                    grant_ids=new_grant_ids
+                )
+                logger.info(f"Generated summaries for {summaries_generated} grants")
+            except Exception as e:
+                logger.error(f"Summary generation failed: {e}", exc_info=True)
+
+        # Step 3: Check for duplicates
+        duplicates = deduplicate_grants(SNAPSHOTS_DIR)
+
+        # Step 4: Mark closed grants
+        closed = mark_closed_grants(SNAPSHOTS_DIR)
+
+        # Step 5: Rebuild index
+        index_rebuilt = rebuild_search_index()
+
+        result = CrawlResult(
+            timestamp=start_time,
+            new_grants=new_grants,
+            updated_grants=summaries_generated,  # Track summaries in updated_grants
+            closed_grants=closed,
+            duplicates_removed=duplicates,
+            index_rebuilt=index_rebuilt,
+        )
+
+        logger.info(f"Crawl cycle complete: {result}")
+        return result
+
+    except Exception as e:
+        logger.error(f"Crawl cycle failed: {e}", exc_info=True)
+        return CrawlResult(
+            timestamp=start_time,
+            new_grants=0,
+            updated_grants=0,
+            closed_grants=0,
+            duplicates_removed=0,
+            index_rebuilt=False,
+            error=str(e)
+        )
+
+
+def setup_scheduler():
+    """
+    Set up APScheduler for daily crawls.
+
+    Usage:
+        from analyzer.crawler.scheduler import setup_scheduler
+        scheduler = setup_scheduler()
+        scheduler.start()
+    """
+    try:
+        from apscheduler.schedulers.background import BackgroundScheduler
+        from apscheduler.triggers.cron import CronTrigger
+
+        if not CRAWL_ENABLED:
+            logger.info("Crawl scheduler disabled (set CRAWL_ENABLED=true to enable)")
+            return None
+
+        scheduler = BackgroundScheduler()
+
+        # Schedule daily crawl at DEFAULT_CRAWL_HOUR (default 2 AM)
+        trigger = CronTrigger(hour=DEFAULT_CRAWL_HOUR, minute=0)
+        scheduler.add_job(
+            run_crawl_cycle,
+            trigger=trigger,
+            id="grant_crawler",
+            name="Daily grant crawl and index refresh",
+            replace_existing=True
+        )
+
+        logger.info(
+            f"Scheduler configured: Daily crawl at {DEFAULT_CRAWL_HOUR}:00 "
+            f"(set CRAWL_ENABLED=true to start)"
+        )
+
+        return scheduler
+
+    except ImportError:
+        logger.warning(
+            "APScheduler not installed. "
+            "Install with: pip install apscheduler"
+        )
+        return None
+
+
+def register_cron_job():
+    """
+    Register a system cron job for daily crawls.
+
+    Useful as alternative to APScheduler for production deployments.
+
+    Example cron line (runs daily at 2 AM):
+    0 2 * * * cd /path/to/grant-analyst && python -m analyzer.crawler.scheduler
+    """
+    import subprocess
+    import platform
+
+    if platform.system() == "Windows":
+        logger.warning("Cron registration only supported on Unix-like systems")
+        return False
+
+    try:
+        script_path = Path(__file__).parent.parent.parent / "crawler" / "scheduler.py"
+        cron_line = f"0 {DEFAULT_CRAWL_HOUR} * * * python {script_path}"
+
+        # This is a guide - actual registration depends on system setup
+        logger.info(f"Add this to crontab for daily crawls:\n{cron_line}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Failed to register cron job: {e}")
+        return False
+
+
+# ============================================================================
+# Entry Points
+# ============================================================================
+
+if __name__ == "__main__":
+    import sys
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+
+    # Run single crawl cycle
+    result = run_crawl_cycle()
+    print(f"\nCrawl Result: {result.to_dict()}")
+
+    if result.error:
+        sys.exit(1)
+    sys.exit(0)

analyzer/crawler/snapshot.py

@@ -0,0 +1,626 @@
+import json
+import re
+import pathlib
+import asyncio
+from urllib.parse import urlparse
+from datetime import datetime, UTC
+from typing import List, Optional, Tuple, Dict
+from playwright.async_api import async_playwright
+from bs4 import BeautifulSoup, Tag, NavigableString
+import typer
+
+# ---------------------------------------------------------------------------
+# HELPERS
+# ---------------------------------------------------------------------------
+
+EXPECTED_TITLES = [
+    "Summary",
+    "Eligibility",
+    "Scope",
+    "Dates",
+    "How to apply",
+    "Supporting information",
+]
+
+# Accept close-enough labels and alias them to canonical 6
+SECTION_ALIASES = {
+    "who can apply": "Eligibility",
+    "who’s eligible": "Eligibility",
+    "who is eligible": "Eligibility",
+    "applicant eligibility": "Eligibility",
+    "what we ask you": "How to apply",
+    "apply": "How to apply",
+    "application process": "How to apply",
+    "supporting info": "Supporting information",
+    "key dates": "Dates",
+    "timeline": "Dates",
+    "competition dates": "Dates",
+    "overview": "Summary",
+    "summary": "Summary",
+    "scope": "Scope",
+}
+
+def canonical_label(label: str) -> Optional[str]:
+    l = label.strip().lower()
+    for t in EXPECTED_TITLES:
+        if l == t.lower():
+            return t
+    return SECTION_ALIASES.get(l, None)
+
+def norm_key(label: str) -> str:
+    return re.sub(r"[^\w\s-]", "", label).strip().lower().replace(" ", "_") + "_raw"
+
+def parse_deeplink(url: str):
+    u = urlparse(url)
+    m = re.search(r"/competition/(\d+)/overview/([0-9a-f-]{8,})", u.path, re.I)
+    if not m:
+        raise ValueError("Expected: .../competition/{id}/overview/{uuid}")
+    return f"{u.scheme}://{u.netloc}", m.group(1), m.group(2)
+
+def get_competition_nav_anchors(soup: BeautifulSoup) -> List[tuple[str, str]]:
+    anchors: List[tuple[str, str]] = []
+
+    headings = soup.find_all(["h2", "h3", "h4"], string=lambda s: isinstance(s, str) and "competition sections" in s.lower())
+    nav_root: Optional[Tag] = None
+    for h in headings:
+        for sib in h.next_siblings:
+            if isinstance(sib, Tag) and sib.name in ("nav", "ul", "ol", "div"):
+                nav_root = sib
+                break
+        if nav_root:
+            break
+
+    if not nav_root:
+        for candidate in soup.find_all("nav"):
+            if candidate.find("a", href=True):
+                nav_root = candidate
+                break
+
+    if nav_root:
+        seen = set()
+        for a in nav_root.find_all("a", href=True):
+            href = a.get("href", "")
+            if not href.startswith("#"):
+                continue
+            frag = href[1:].strip()
+            raw = (a.get_text(" ", strip=True) or "").strip()
+            if not frag or not raw:
+                continue
+            canon = canonical_label(raw) or raw
+            if canon in EXPECTED_TITLES and frag not in seen:
+                anchors.append((canon, frag))
+                seen.add(frag)
+
+    if not anchors:
+        anchors = [
+            ("Summary", "summary"),
+            ("Eligibility", "eligibility"),
+            ("Scope", "scope"),
+            ("Dates", "dates"),
+            ("How to apply", "how-to-apply"),
+            ("Supporting information", "supporting-information"),
+        ]
+    return anchors
+
+# -----------------------------
+# Footer / cookie / consent trimmer
+# -----------------------------
+
+_FOOTER_STOPS = [
+    "Need help with this service?",
+    "Support links",
+    "GOV.UK uses cookies",
+    "Create one update function for each consent parameter",
+    "© Crown copyright",
+    "All content is available under the Open Government Licence",
+]
+
+def trim_footer(text: str) -> str:
+    if not text:
+        return text
+    for marker in _FOOTER_STOPS:
+        i = text.find(marker)
+        if i != -1:
+            return text[:i].rstrip()
+    return text
+
+def _strip_boilerplate(soup: BeautifulSoup):
+    selectors = [
+        "#global-cookie-message", ".cookie-banner", "#ccc-notify", "#onetrust-banner-sdk",
+        "footer", ".govuk-footer",
+        ".govuk-prototype-kit-warning",
+    ]
+    for sel in selectors:
+        for el in soup.select(sel):
+            el.decompose()
+
+# ---------------------------------------------------------------------------
+# TITLE
+# ---------------------------------------------------------------------------
+
+def clean_title(raw: str) -> str:
+    if not raw:
+        return raw
+    raw = raw.strip()
+    return re.sub(r"^\s*Funding competition\s+", "", raw, flags=re.I).strip()
+
+def extract_between_ids(soup: BeautifulSoup, start_id: str, end_id: Optional[str]) -> str:
+    start = soup.find(id=start_id)
+    if not start:
+        return ""
+    out_chunks: List[str] = []
+    for el in start.next_elements:
+        if isinstance(el, Tag):
+            if end_id and el.get("id") == end_id:
+                break
+            if el.name in ("script", "style", "noscript"):
+                continue
+        if isinstance(el, NavigableString):
+            txt = el.strip()
+            if txt:
+                out_chunks.append(txt)
+    text = " ".join(out_chunks)
+    text = re.sub(r"\s+", " ", text).strip()
+    text = trim_footer(text)
+    return text
+
+# ---------------------------------------------------------------------------
+# PARSING
+# ---------------------------------------------------------------------------
+
+def slice_by_anchors(html: str) -> dict:
+    soup = BeautifulSoup(html, "html.parser")
+    _strip_boilerplate(soup)
+    anchors = get_competition_nav_anchors(soup)
+    ids_in_order = [aid for _, aid in anchors]
+    id_to_next = {ids_in_order[i]: (ids_in_order[i + 1] if i + 1 < len(ids_in_order) else None)
+                  for i in range(len(ids_in_order))}
+    out = {}
+    for label, start_id in anchors:
+        next_id = id_to_next.get(start_id)
+        key = norm_key(label)
+        out[key] = extract_between_ids(soup, start_id, next_id)
+    return out
+
+# ---------------------------------------------------------------------------
+# MAIN SCRAPER
+# ---------------------------------------------------------------------------
+
+async def fetch_sections_from_overview(url: str) -> tuple[str, dict]:
+    scheme_host, comp_id, uuid = parse_deeplink(url)
+    overview_url = f"{scheme_host}/competition/{comp_id}/overview/{uuid}"
+
+    async with async_playwright() as pw:
+        browser = await pw.chromium.launch(headless=True, args=["--disable-dev-shm-usage"])
+        context = await browser.new_context(
+            user_agent=("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
+            locale="en-GB",
+            timezone_id="Europe/London",
+        )
+        page = await context.new_page()
+        page.set_default_timeout(30000)
+
+        for attempt in range(2):
+            try:
+                await page.goto(overview_url, wait_until="domcontentloaded")
+                await page.wait_for_load_state("networkidle")
+                break
+            except Exception:
+                if attempt == 1:
+                    raise
+        html = await page.content()
+        await context.close()
+        await browser.close()
+
+    return html, slice_by_anchors(html)
+
+# ---------------------------------------------------------------------------
+# DATE PARSING — SINGLE-LINE CHUNKS
+# ---------------------------------------------------------------------------
+
+# tokens
+_DATE_WORD = r"(?:\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4}|[A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4})"
+_TIME_WORD = r"(?:\d{1,2}:\d{2}\s*[ap]m|\d{1,2}\s*[ap]m)"
+
+_DATE_ONLY_RX = re.compile(_DATE_WORD, re.I)
+_TIME_RX = re.compile(_TIME_WORD, re.I)
+
+# e.g. "9 to 20 March 2026", "9–20 March 2026", "9 - 20 March 2026"
+_DATE_RANGE_RX = re.compile(r"(\d{1,2})\s*(?:to|-|–)\s*(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", re.I)
+
+_MONTHS = {m.lower(): i for i, m in enumerate(
+    ["January","February","March","April","May","June","July","August","September","October","November","December"], 1
+)}
+
+_EXCLUDE_SENTENCE_CUES = ["briefing event", "briefing", "webinar", "register to attend", "register", "info session"]
+
+_LABEL_RULES = [
+    ("opens", ["competition opens", "opens"]),
+    ("closes", ["competition closes", "closes", "deadline"]),
+    ("notify", ["applicants notified", "applicants will be notified", "notification"]),
+    ("project_start", ["project start from", "project starts from", "project start date", "project start"]),
+    ("assessment", ["interview", "assessment", "panel"]),
+    ("results", ["results published", "winners announced"]),
+    ("eligibility_cutoff", ["eligibility closes", "registration closes"]),
+    ("info_session", ["briefing", "webinar", "register"]),
+]
+
+def _classify_label(sent_lower: str) -> str:
+    for norm, cues in _LABEL_RULES:
+        if any(c in sent_lower for c in cues):
+            return norm
+    return "other"
+
+def _parse_single_date(token: str, time_hint: Optional[str]) -> Optional[str]:
+    token = token.strip()
+    m_comma = re.match(r"([A-Za-z]{3,9})\s+(\d{1,2}),?\s+(\d{4})", token)
+    if m_comma:
+        month_name, day, year = m_comma.groups()
+    else:
+        m = re.match(r"(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", token)
+        if not m:
+            return None
+        day, month_name, year = m.groups()
+    month = _MONTHS.get(month_name.lower())
+    if not month:
+        return None
+    if time_hint:
+        t = time_hint.lower().replace(" ", "")
+        mm = re.match(r"(\d{1,2})(?::(\d{2}))?([ap]m)", t)
+        if mm:
+            hh = int(mm.group(1))
+            mins = int(mm.group(2) or 0)
+            ampm = mm.group(3)
+            if ampm == "pm" and hh != 12: hh += 12
+            if ampm == "am" and hh == 12: hh = 0
+            return f"{int(year):04d}-{month:02d}-{int(day):02d}T{hh:02d}:{mins:02d}:00"
+    return f"{int(year):04d}-{month:02d}-{int(day):02d}"
+
+def parse_dates_singleline(text: str) -> List[dict]:
+    """
+    Split the Dates section into *one milestone per line/chunk*:
+      chunk := from each DATE token up to the next DATE token (or end).
+    Keeps the entire chunk in label_raw.
+    """
+    milestones: List[dict] = []
+    if not text:
+        return milestones
+
+    # find all date token positions
+    matches = list(_DATE_ONLY_RX.finditer(text))
+    if not matches:
+        return milestones
+
+    spans = []
+    for i, m in enumerate(matches):
+        start = m.start()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        spans.append((start, end))
+
+    for (start, end) in spans:
+        chunk = text[start:end].strip()
+        if not chunk:
+            continue
+        low = chunk.lower()
+        excluded = any(k in low for k in _EXCLUDE_SENTENCE_CUES)
+
+        # primary date + optional time in this chunk
+        first_date = _DATE_ONLY_RX.search(chunk)
+        time_hint_match = _TIME_RX.search(chunk)
+        iso = _parse_single_date(first_date.group(0), time_hint_match.group(0) if time_hint_match else None) if first_date else None
+
+        # optional same-month day range inside the chunk
+        r = _DATE_RANGE_RX.search(chunk)
+        date_end_iso = None
+        if r:
+            d1, d2, mon_name, year = r.groups()
+            month = _MONTHS.get(mon_name.lower())
+            if month:
+                date_end_iso = f"{int(year):04d}-{month:02d}-{int(d2):02d}"
+                # if the start of the range equals first_date, keep iso as start;
+                # otherwise we still keep iso from first_date (which begins the chunk)
+
+        if iso:
+            milestones.append({
+                "label_raw": chunk,
+                "label_norm": _classify_label(low),
+                "date_iso": iso,
+                "date_iso_end": date_end_iso,
+                "has_time": bool(time_hint_match),
+                "excluded_from_open_close": excluded,
+            })
+
+    return milestones
+
+def pick_open_close_from_milestones(milestones: List[dict]) -> Tuple[Optional[str], Optional[str]]:
+    open_iso = close_iso = None
+    for m in milestones:
+        if m["excluded_from_open_close"]:
+            continue
+        if m["label_norm"] == "opens" and open_iso is None:
+            open_iso = m["date_iso"].split("T")[0]
+        if m["label_norm"] == "closes" and close_iso is None:
+            close_iso = m["date_iso"]
+    if open_iso is None:
+        for m in milestones:
+            if m["label_norm"] == "opens":
+                open_iso = m["date_iso"].split("T")[0]
+                break
+    if close_iso is None:
+        for m in milestones:
+            if m["label_norm"] == "closes":
+                close_iso = m["date_iso"]
+                break
+    return open_iso, close_iso
+
+def derive_aux_dates(milestones: List[dict]) -> Tuple[Optional[str], Optional[str]]:
+    notify = project_start_from = None
+    for m in milestones:
+        if notify is None and m["label_norm"] == "notify":
+            notify = m["date_iso"].split("T")[0]
+        if project_start_from is None and m["label_norm"] == "project_start":
+            project_start_from = m["date_iso"].split("T")[0]
+        if notify and project_start_from:
+            break
+    return notify, project_start_from
+
+# ---------------------------------------------------------------------------
+# FUNDING / COMPENSATION PARSING
+# ---------------------------------------------------------------------------
+
+_MONEY_TOKEN = re.compile(r"(£|\bGBP\s*)([\d,]+(?:\.\d+)?)(?:\s*(million|m|billion|bn|k))?", re.I)
+
+def _money_to_int(sign: str, num_str: str, mag: Optional[str]) -> int:
+    val = float(num_str.replace(",", ""))
+    if mag:
+        m = mag.lower()
+        if m in ("million", "m"):
+            val *= 1_000_000
+        elif m in ("billion", "bn"):
+            val *= 1_000_000_000
+        elif m in ("k",):
+            val *= 1_000
+    return int(round(val))
+
+_TOTAL_CUES = [
+    "total prize fund", "total prize pot", "total funding available", "available in total",
+    "total pot", "prize fund", "funding pot", "overall budget", "total budget",
+    "total allocation", "in total across", "total amount available",
+]
+_AWARD_CUES = [
+    "per project", "each project", "you can apply for", "can apply for", "apply for up to",
+    "grant of up to", "awards of up to", "awards between", "awards of between",
+    "fund between", "we will fund", "we can fund", "project costs between",
+    "total eligible project costs between", "your project must have total costs between",
+    "maximum grant", "minimum grant", "maximum funding", "minimum funding",
+    "up to", "no more than", "at least",
+    "grant funding request", "eligible grant funding", "eligible grant", "funding request must be between",
+]
+_EXCLUDE_CUES = [
+    "market", "industry", "global", "worldwide", "valuation", "addressable", "gdp",
+    "economy", "sector value", "turnover", "revenue", "jobs", "headcount",
+]
+
+_RANGE_PATTERNS = [
+    re.compile(rf"(?:between|from)\s+{_MONEY_TOKEN.pattern}\s+(?:and|to)\s+{_MONEY_TOKEN.pattern}", re.I),
+    re.compile(rf"{_MONEY_TOKEN.pattern}\s*(?:to|-)\s*{_MONEY_TOKEN.pattern}", re.I),
+]
+_MAX_PATTERNS = [
+    re.compile(rf"(?:up to|no more than|max(?:imum)?(?:\s+grant|\s+funding)?)\s+{_MONEY_TOKEN.pattern}", re.I),
+]
+_MIN_PATTERNS = [
+    re.compile(rf"(?:at least|minimum(?:\s+grant|\s+funding)?)\s+{_MONEY_TOKEN.pattern}", re.I),
+]
+
+def _contains_any(text: str, cues: List[str]) -> bool:
+    low = text.lower()
+    return any(c in low for c in cues)
+
+def _is_excluded_sentence(sent: str) -> bool:
+    return _contains_any(sent, _EXCLUDE_CUES)
+
+def _split_sentences_generic(text: str) -> List[str]:
+    parts = re.split(r"(?:\n+|(?<=[\.\!\?])\s+)", text)
+    return [p.strip() for p in parts if p and p.strip()]
+
+def _find_total_pot(text: str) -> Optional[int]:
+    if not text:
+        return None
+    best = None
+    for sent in _split_sentences_generic(text):
+        if _is_excluded_sentence(sent):
+            continue
+        if _contains_any(sent, _TOTAL_CUES):
+            vals = []
+            for m in _MONEY_TOKEN.finditer(sent):
+                _, num_str, mag = m.groups()
+                vals.append(_money_to_int("£", num_str, mag))
+            if vals:
+                v = max(vals)
+                best = v if best is None or v > best else best
+    return best
+
+def _find_award_range(text: str) -> Tuple[Optional[int], Optional[int]]:
+    if not text:
+        return None, None
+
+    # Strong: explicit ranges in a sentence that has award cues
+    for sent in _split_sentences_generic(text):
+        if _is_excluded_sentence(sent):
+            continue
+        if not _contains_any(sent, _AWARD_CUES):
+            continue
+        for rx in _RANGE_PATTERNS:
+            m = rx.search(sent)
+            if not m:
+                continue
+            monies = list(_MONEY_TOKEN.finditer(m.group(0)))
+            if len(monies) >= 2:
+                v1 = _money_to_int(*("£", monies[-2].group(2), monies[-2].group(3)))
+                v2 = _money_to_int(*("£", monies[-1].group(2), monies[-1].group(3)))
+                lo, hi = sorted([v1, v2])
+                return lo, hi
+
+    # Next: max-only / min-only with cues
+    chosen_min = None
+    chosen_max = None
+    for sent in _split_sentences_generic(text):
+        if _is_excluded_sentence(sent):
+            continue
+        if not _contains_any(sent, _AWARD_CUES):
+            continue
+
+        if chosen_max is None:
+            for rx in _MAX_PATTERNS:
+                m = rx.search(sent)
+                if m:
+                    money = _MONEY_TOKEN.search(m.group(0))
+                    if money:
+                        chosen_max = _money_to_int(*("£", money.group(2), money.group(3)))
+                        break
+
+        if chosen_min is None:
+            for rx in _MIN_PATTERNS:
+                m = rx.search(sent)
+                if m:
+                    money = _MONEY_TOKEN.search(m.group(0))
+                    if money:
+                        chosen_min = _money_to_int(*("£", money.group(2), money.group(3)))
+                        break
+
+        if chosen_min is not None and chosen_max is not None:
+            break
+
+    return chosen_min, chosen_max
+
+# NEW: funding rates & duration
+_RATE_LINE = re.compile(
+    r"up to\s*(\d{1,3})%\s*if you are a\s*(?:micro|small).*?up to\s*(\d{1,3})%\s*if you are a\s*medium.*?up to\s*(\d{1,3})%\s*if you are a\s*large",
+    re.I | re.S,
+)
+def _find_funding_rates(text: str) -> Optional[dict]:
+    if not text:
+        return None
+    m = _RATE_LINE.search(text)
+    if not m:
+        return None
+    small, medium, large = map(int, m.groups())
+    return {"micro_small": small, "medium": medium, "large": large}
+
+_DURATION_RX = re.compile(r"last\s+between\s+(\d{1,3})\s*(?:and|to|–|-)\s*(\d{1,3})\s+months", re.I)
+def _find_duration_months(text: str) -> Tuple[Optional[int], Optional[int]]:
+    if not text:
+        return None, None
+    m = _DURATION_RX.search(text)
+    if not m:
+        return None, None
+    lo, hi = map(int, m.groups())
+    return (lo if lo <= hi else hi), (hi if hi >= lo else lo)
+
+def extract_funding(sections: dict) -> dict:
+    summary = sections.get("summary_raw", "") or ""
+    support = sections.get("supporting_information_raw", "") or ""
+    scope = sections.get("scope_raw", "") or ""
+    eligibility = sections.get("eligibility_raw", "") or ""
+    all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
+
+    total_pot = _find_total_pot(summary) or _find_total_pot(support) or _find_total_pot(all_text)
+
+    min_award = max_award = None
+    for candidate in (summary, eligibility, support, scope, all_text):
+        lo, hi = _find_award_range(candidate)
+        if lo is not None or hi is not None:
+            if lo is not None: min_award = lo
+            if hi is not None: max_award = hi
+            break
+
+    rates = None
+    for candidate in (eligibility, support, all_text):
+        rates = _find_funding_rates(candidate or "")
+        if rates:
+            break
+
+    return {"min": min_award, "max": max_award, "total_pot": total_pot, "rates": rates}
+
+# ---------------------------------------------------------------------------
+# MAIN
+# ---------------------------------------------------------------------------
+
+async def _main_async(url: str):
+    m = re.search(r"/competition/(\d+)", url)
+    slug = f"competition-{m.group(1)}" if m else re.sub(r"[^a-z0-9]+", "-", url.lower()).strip("-")[-60:]
+    out_path = pathlib.Path("data/snapshots") / f"{slug}.json"
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    html, sections = await fetch_sections_from_overview(url)
+    soup = BeautifulSoup(html, "html.parser")
+    h1 = soup.find("h1")
+    raw_title = h1.get_text(" ", strip=True) if h1 else ""
+    title = clean_title(raw_title)
+
+    # Dates -> single-line chunks
+    dates_text = sections.get("dates_raw", "") or ""
+    milestones = parse_dates_singleline(dates_text)
+
+    # Derive open/close from milestones (with exclusions for briefing lines)
+    open_date, close_date = pick_open_close_from_milestones(milestones)
+
+    # If either missing, try scanning all text but still as single-line chunks
+    if not open_date or not close_date:
+        all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
+        extra = parse_dates_singleline(all_text)
+        # merge de-duped by (label_raw, date_iso, date_iso_end)
+        seen = {(m["label_raw"], m["date_iso"], m.get("date_iso_end")) for m in milestones}
+        for m2 in extra:
+            key = (m2["label_raw"], m2["date_iso"], m2.get("date_iso_end"))
+            if key not in seen:
+                milestones.append(m2)
+                seen.add(key)
+        od2, cd2 = pick_open_close_from_milestones(milestones)
+        open_date = open_date or od2
+        close_date = close_date or cd2
+
+    notify_date, project_start_from = derive_aux_dates(milestones)
+
+    # Funding
+    funding = extract_funding(sections)
+
+    # Duration months
+    dur_min, dur_max = _find_duration_months(sections.get("eligibility_raw", "") or "")
+    if dur_min is None or dur_max is None:
+        all_text_for_duration = " ".join(v for v in sections.values() if isinstance(v, str) and v)
+        dur_min, dur_max = _find_duration_months(all_text_for_duration)
+    duration_months = {"min": dur_min, "max": dur_max}
+
+    snapshot = {
+        "url": url,
+        "title": title,
+        "programme": "",
+        "round": "",
+        "open_date": open_date,
+        "close_date": close_date,
+        "notify_date": notify_date,
+        "project_start_from": project_start_from,
+        "funding": funding,
+        "duration_months": duration_months,
+        "sections": sections,
+        "pdfs": [],
+        "summaries": {},
+        "extracted": {"milestones": milestones},
+        "wonky": {"score": 0.0, "reasons": []},
+        "prev_round_refs": [],
+        "diff_summary": "",
+        "history_stats": {},
+        "created_at": datetime.now(UTC).isoformat(),
+        "updated_at": datetime.now(UTC).isoformat(),
+    }
+
+    out_path.write_text(json.dumps(snapshot, indent=2), encoding="utf-8")
+    print(f"Snapshot saved to {out_path}")
+
+def main(url: str = typer.Argument(..., help="IFS overview URL e.g. .../competition/{id}/overview/{uuid}")):
+    asyncio.run(_main_async(url))
+
+if __name__ == "__main__":
+    typer.run(main)

analyzer/data_loader.py

@@ -0,0 +1,167 @@
+"""
+data_loader.py — loads current grant snapshots (JSON) and optional past winners
+
+This module keeps IO concerns simple and robust:
+- Recursively loads current-grant JSON files under a snapshots directory
+- Optionally loads past winners from either an Excel file or a JSON directory
+- Returns Python lists of dictionaries; no model code here
+
+Public API
+---------
+load_current_grants(snapshots_dir: Path | str, limit: int | None = None) -> list[dict]
+load_past_winners(history_xlsx: Path | str | None = None,
+                  history_json_dir: Path | str | None = None) -> list[dict]
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+import json
+import logging
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+# ----------------------------- Current grants ---------------------------------
+
+def load_current_grants(snapshots_dir: Path | str, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+    """Load current grant JSON snapshots from a directory tree.
+
+    Each file is expected to be one JSON object. The function tolerates
+    missing keys and will attach an `id` from the filename if not present.
+    """
+    snapshots_dir = Path(snapshots_dir)
+    if not snapshots_dir.exists():
+        logger.warning("snapshots directory not found: %s", snapshots_dir)
+        return []
+
+    records: List[Dict[str, Any]] = []
+    for p in sorted(snapshots_dir.rglob("*.json")):
+        try:
+            with open(p, "r", encoding="utf-8") as f:
+                rec = json.load(f)
+            if not isinstance(rec, dict):
+                logger.debug("Skipping non-object JSON: %s", p)
+                continue
+            rec.setdefault("id", p.stem)
+            rec.setdefault("_path", str(p))
+            records.append(rec)
+            if limit and len(records) >= limit:
+                break
+        except Exception as e:  # pragma: no cover
+            logger.warning("Failed to load %s: %s", p, e)
+            continue
+
+    logger.info("Loaded %d current grants from %s", len(records), snapshots_dir)
+    return records
+
+# ------------------------------ Past winners ----------------------------------
+
+_CANON_COLS = {
+    # canonical : candidate column names (lower/underscore)
+    "project_title": ["project_title", "title", "name"],
+    "abstract": ["abstract", "description", "summary", "public_description"],
+    "competition": ["competition", "programme", "program"],
+    "award_amount": ["award_amount", "amount", "grant", "project_cost", "award"],
+    "lead_org": ["lead_org", "lead_organisation", "lead_organization", "organisation_name", "organization_name"],
+    "year": ["year", "fy", "start_year"],
+    "project_url": ["project_url", "url", "link"],
+}
+
+
+def _norm_cols(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+    df.columns = [
+        (c if isinstance(c, str) else str(c))
+        .lower()
+        .replace(" ", "_")
+        .replace("-", "_")
+        for c in df.columns
+    ]
+    return df
+
+
+def _ensure_canonical(df: pd.DataFrame) -> pd.DataFrame:
+    for canon, candidates in _CANON_COLS.items():
+        if canon in df.columns:
+            continue
+        for c in candidates:
+            if c in df.columns:
+                df[canon] = df[c]
+                break
+        if canon not in df.columns:
+            df[canon] = None
+    return df
+
+
+def _load_past_winners_from_excel(xlsx_path: Path) -> List[Dict[str, Any]]:
+    df = pd.read_excel(xlsx_path)
+    df = _norm_cols(df)
+    df = _ensure_canonical(df)
+    return [row._asdict() if hasattr(row, "_asdict") else row.to_dict() for _, row in df.iterrows()]
+
+
+def _load_past_winners_from_json_dir(json_dir: Path) -> List[Dict[str, Any]]:
+    records: List[Dict[str, Any]] = []
+    for p in sorted(json_dir.rglob("*.json")):
+        try:
+            with open(p, "r", encoding="utf-8") as f:
+                rec = json.load(f)
+            if not isinstance(rec, dict):
+                continue
+            rec.setdefault("_path", str(p))
+            records.append(rec)
+        except Exception:
+            continue
+    return records
+
+
+def load_past_winners(history_xlsx: Path | str | None = None,
+                      history_json_dir: Path | str | None = None) -> List[Dict[str, Any]]:
+    """Load past winners from either Excel (preferred) or a JSON folder.
+
+    Priority order: JSON dir (if provided) > Excel (if provided).
+    If neither exists, returns an empty list.
+    """
+    # JSON dir first if present
+    if history_json_dir is not None:
+        jdir = Path(history_json_dir)
+        if jdir.exists():
+            recs = _load_past_winners_from_json_dir(jdir)
+            if recs:
+                logger.info("Loaded %d past winners from JSON dir: %s", len(recs), jdir)
+                return recs
+            else:
+                logger.info("No JSON past winners found under %s", jdir)
+
+    # Excel next
+    if history_xlsx is not None:
+        xlsx = Path(history_xlsx)
+        if xlsx.exists():
+            recs = _load_past_winners_from_excel(xlsx)
+            logger.info("Loaded %d past winners from Excel: %s", len(recs), xlsx)
+            return recs
+        else:
+            logger.info("History Excel not found: %s", xlsx)
+
+    return []
+
+
+# Self-test
+if __name__ == "__main__":
+    import argparse
+    logging.basicConfig(level=logging.INFO)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--snapshots-dir", type=Path, default=Path("data/snapshots"))
+    parser.add_argument("--history-xlsx", type=Path, default=Path("data/past_winners.xlsx"))
+    parser.add_argument("--limit", type=int, default=3)
+    args = parser.parse_args()
+
+    current = load_current_grants(args.snapshots_dir, limit=args.limit)
+    history = load_past_winners(args.history_xlsx)
+    print(f"current: {len(current)} | history: {len(history)}")
+    if current:
+        print("example current keys:", sorted(current[0].keys())[:12])
+    if history:
+        print("example history keys:", sorted(history[0].keys())[:12])

analyzer/data_loader_supporting.py

@@ -0,0 +1,163 @@
+# src/analyzer/data_loader_supporting.py
+from __future__ import annotations
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional, Iterable, Union
+import json, re
+
+from .utils.text import to_number
+
+@dataclass
+class SupportingDoc:
+    grant_id: str
+    url: str
+    title: str
+    open_date: Optional[str]
+    close_date: Optional[str]
+    notify_date: Optional[str]
+    funding_min: Optional[float]
+    funding_max: Optional[float]
+    total_pot: Optional[float]
+    funding_rates: Optional[str]
+    duration_min: Optional[int]
+    duration_max: Optional[int]
+    text: str                    # flattened blob for retrieval
+    sections: Dict[str, str]     # raw sections, if present
+
+# Number parsing moved to utils.text.to_number()
+# Keeping wrapper for backward compatibility
+def _num(x):
+    return to_number(x)
+
+def _int(x):
+    try:
+        return int(x) if x is not None else None
+    except Exception:
+        try:
+            return int(float(str(x).replace(",", "")))
+        except Exception:
+            return None
+
+def _infer_grant_id(obj: dict, fallback_name: str = "") -> Optional[str]:
+    # 1) explicit fields
+    for k in ("grant_id","id","competition_id","competitionId"):
+        if obj.get(k):
+            return str(obj[k]).replace("competition-","").strip()
+    # 2) from URL: .../competition/2185/...
+    url = obj.get("url") or obj.get("source_url") or obj.get("page_url") or ""
+    m = re.search(r"/competition/(\d+)", url)
+    if m:
+        return m.group(1)
+    # 3) from filename
+    m2 = re.search(r"competition-(\d+)", fallback_name)
+    if m2:
+        return m2.group(1)
+    return None
+
+def _make_text_blob(title: str, url: str, sections: Dict[str,str]) -> str:
+    parts = [f"TITLE: {title}", f"URL: {url}"]
+    for k in ("summary_raw","eligibility_raw","scope_raw","dates_raw","how_to_apply_raw","supporting_information_raw"):
+        v = sections.get(k)
+        if v:
+            parts.append(f"\n[{k}]\n{v}")
+    # also tolerate alt keys from other crawlers
+    for k in ("summary","eligibility","scope","dates","how_to_apply","supporting_information"):
+        v = sections.get(k)
+        if v and f"[{k}_raw]" not in "".join(parts):
+            parts.append(f"\n[{k}]\n{v}")
+    return "\n".join(parts)
+
+def _read_obj(obj: dict, fallback_name: str = "") -> Optional[SupportingDoc]:
+    gid = _infer_grant_id(obj, fallback_name)
+    if not gid:
+        return None
+
+    url = obj.get("url") or obj.get("source_url") or obj.get("page_url") or ""
+    title = (obj.get("title") or obj.get("name") or "").strip()
+
+    # Common normalised fields
+    open_date  = obj.get("open_date")
+    close_date = obj.get("close_date") or obj.get("deadline") or obj.get("closeDate")
+    notify_date = obj.get("notify_date")
+
+    # Funding block: either nested or flat
+    funding = obj.get("funding") or {}
+    fmin = _num(funding.get("min") or obj.get("funding_min") or obj.get("min_award") or obj.get("grant_min"))
+    fmax = _num(funding.get("max") or obj.get("funding_max") or obj.get("max_award") or obj.get("grant_max"))
+    total_pot = _num(funding.get("total_pot") or obj.get("total_pot") or obj.get("competition_total") or obj.get("total_funding"))
+    rates = funding.get("rates") if isinstance(funding.get("rates"), str) else obj.get("funding_rates")
+
+    # Duration block
+    dur = obj.get("duration_months") or {}
+    dmin = _int(dur.get("min") or obj.get("duration_min") or obj.get("project_duration_min_months"))
+    dmax = _int(dur.get("max") or obj.get("duration_max") or obj.get("project_duration_max_months"))
+
+    # Sections: tolerate both nested and flat naming
+    sections: Dict[str, str] = {}
+    for k in ("summary_raw","eligibility_raw","scope_raw","dates_raw","how_to_apply_raw","supporting_information_raw",
+              "summary","eligibility","scope","dates","how_to_apply","supporting_information"):
+        v = obj.get(k) or (obj.get("sections") or {}).get(k)
+        if isinstance(v, str) and v.strip():
+            sections[k] = v
+
+    text = _make_text_blob(title, url, sections)
+    return SupportingDoc(
+        grant_id=gid,
+        url=url,
+        title=title,
+        open_date=open_date,
+        close_date=close_date,
+        notify_date=notify_date,
+        funding_min=fmin,
+        funding_max=fmax,
+        total_pot=total_pot,
+        funding_rates=rates if isinstance(rates, str) else None,
+        duration_min=dmin,
+        duration_max=dmax,
+        text=text,
+        sections=sections,
+    )
+
+def _read_json_file(p: Path) -> Optional[SupportingDoc]:
+    try:
+        obj = json.loads(p.read_text(encoding="utf-8"))
+        return _read_obj(obj, fallback_name=p.name)
+    except Exception:
+        return None
+
+def _iter_jsonl(p: Path) -> Iterable[SupportingDoc]:
+    with p.open("r", encoding="utf-8") as f:
+        for i, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except Exception:
+                continue
+            doc = _read_obj(obj, fallback_name=f"{p.name}:{i}")
+            if doc:
+                yield doc
+
+def iter_supporting_docs(folder: Path) -> Iterable[SupportingDoc]:
+    folder = Path(folder)
+    # Prefer explicit competition-*.json first
+    found = False
+    for p in sorted(folder.glob("competition-*.json")):
+        found = True
+        doc = _read_json_file(p)
+        if doc:
+            yield doc
+    # Then any *.json
+    if not found:
+        for p in sorted(folder.glob("*.json")):
+            doc = _read_json_file(p)
+            if doc:
+                yield doc
+    # Then *.jsonl (one object per line)
+    for p in sorted(folder.glob("*.jsonl")):
+        for doc in _iter_jsonl(p):
+            yield doc
+
+def load_supporting_docs(folder: Path) -> List[SupportingDoc]:
+    return list(iter_supporting_docs(folder))

analyzer/exporters.py

@@ -0,0 +1,77 @@
+"""
+exporters.py — save summaries to Excel and JSONL
+
+Public API
+----------
+- export_excel(rows: list[dict], out_path: Path | str) -> None
+- export_jsonl(rows: list[dict], out_path: Path | str) -> None
+
+Each row should minimally include: grant_id, title, summary_md.
+Optional keys like context, source_path will be preserved (JSONL)
+and omitted from the Excel unless you want to include them explicitly.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List
+import json
+import pandas as pd
+
+_EXCEL_COLUMNS = [
+    "grant_id",
+    "title",
+    "summary_md",
+]
+
+
+def export_excel(rows: List[Dict[str, Any]], out_path: Path | str) -> None:
+    out = Path(out_path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+
+    # Keep Excel tidy with a predictable column order; include extras at the end
+    if rows:
+        # determine any extra keys beyond the base set
+        base = set(_EXCEL_COLUMNS)
+        extra_keys = []
+        for r in rows:
+            for k in r.keys():
+                if k not in base and k not in extra_keys:
+                    extra_keys.append(k)
+        cols = _EXCEL_COLUMNS + [k for k in extra_keys if k not in ("context",)]
+    else:
+        cols = _EXCEL_COLUMNS
+
+    df = pd.DataFrame(rows)
+    # Ensure columns exist even if missing in data
+    for c in cols:
+        if c not in df.columns:
+            df[c] = None
+
+    with pd.ExcelWriter(out, engine="xlsxwriter") as writer:
+        df[cols].to_excel(writer, index=False, sheet_name="summaries")
+        ws = writer.sheets["summaries"]
+        # simple width heuristic
+        for idx, col in enumerate(cols):
+            s = df[col].astype(str)
+            width = min(80, max(12, int(s.str.len().quantile(0.9)) + 3))
+            ws.set_column(idx, idx, width)
+
+
+def export_jsonl(rows: List[Dict[str, Any]], out_path: Path | str) -> None:
+    out = Path(out_path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with open(out, "w", encoding="utf-8") as f:
+        for r in rows:
+            json.dump(r, f, ensure_ascii=False)
+            f.write("\n")
+
+
+# Self-test
+if __name__ == "__main__":
+    demo = [
+        {"grant_id": "g1", "title": "AI in MFG", "summary_md": "**Example**"},
+        {"grant_id": "g2", "title": "Net Zero", "summary_md": "text", "source_path": "foo.json"},
+    ]
+    export_excel(demo, "./_out/demo.xlsx")
+    export_jsonl(demo, "./_out/demo.jsonl")
+    print("Wrote ./_out/demo.xlsx and ./_out/demo.jsonl")

analyzer/llm_client.py

@@ -0,0 +1,396 @@
+# src/analyzer/llm_client.py
+from __future__ import annotations
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional, Callable, Literal
+
+from .utils.errors import LLMError, ConfigError
+
+ModelType = Literal["router", "translator", "analyzer"]
+VerbosityLevel = Literal["low", "medium", "high"]
+ReasoningEffort = Literal["minimal", "medium", "high"]
+
+try:
+    from openai import OpenAI
+    import httpx
+except ImportError:
+    OpenAI = None
+    httpx = None
+
+logger = logging.getLogger(__name__)
+
+
+class LLMClient:
+    """
+    Lightweight wrapper around OpenAI Chat Completions with:
+      - per-call max_tokens
+      - retry with exponential backoff
+      - proper error handling and timeouts
+    """
+
+    def __init__(self, cfg: Any):
+        # Extract config (supports both dict and object)
+        self.provider = self._get_cfg(cfg, "provider", "openai")
+        self.model = self._get_cfg(cfg, "model", "gpt-5-mini")
+        self.disable_llm = self._get_cfg(cfg, "disable_llm", False)
+
+        # Store model variants for different use cases
+        self.model_router = self._get_cfg(cfg, "model_router", "gpt-5-nano")
+        self.model_translator = self._get_cfg(cfg, "model_translator", "gpt-5-mini")
+        self.model_analyzer = self._get_cfg(cfg, "model_analyzer", "gpt-5")
+
+        api_key = self._get_cfg(cfg, "api_key") or os.getenv("OPENAI_API_KEY")
+        base_url = self._get_cfg(cfg, "base_url") or os.getenv(
+            "OPENAI_API_BASE",
+            "https://api.openai.com/v1"
+        )
+
+        self.max_retries = int(os.getenv("LLM_MAX_RETRIES", "3"))
+        self.retry_delay = float(os.getenv("LLM_RETRY_DELAY", "1.0"))
+        self.client = None
+
+        # Validate provider
+        if self.provider not in ("openai", "anthropic"):
+            raise ConfigError(
+                f"Unsupported LLM provider: {self.provider}. "
+                f"Supported: openai, anthropic"
+            )
+
+        # Initialize client (if not disabled)
+        if self.disable_llm:
+            logger.info("LLM disabled via config")
+            return
+
+        if self.provider != "openai":
+            logger.warning("Provider %s not yet fully supported", self.provider)
+            return
+
+        if OpenAI is None:
+            raise ConfigError(
+                "openai package not installed. Install with: pip install openai"
+            )
+
+        if not api_key:
+            raise ConfigError(
+                "OpenAI API key required. Set OPENAI_API_KEY environment variable."
+            )
+
+        try:
+            self.client = OpenAI(api_key=api_key, base_url=base_url)
+            logger.info("LLMClient initialized: %s/%s", self.provider, self.model)
+        except Exception as e:
+            raise ConfigError(f"Failed to initialize OpenAI client: {e}") from e
+
+    def _get_cfg(self, cfg: Any, key: str, default: Any = None) -> Any:
+        """Extract config value (works with both dict and object)."""
+        if hasattr(cfg, key):
+            return getattr(cfg, key) or default
+        if isinstance(cfg, dict):
+            return cfg.get(key, default)
+        return default
+
+    def is_ready(self) -> bool:
+        """Check if LLM client is ready to use."""
+        return self.client is not None and not self.disable_llm
+
+    def _get_model_for_type(self, model_type: Optional[ModelType] = None) -> str:
+        """Get the appropriate model for the given type."""
+        if model_type == "router":
+            return self.model_router
+        elif model_type == "translator":
+            return self.model_translator
+        elif model_type == "analyzer":
+            return self.model_analyzer
+        else:
+            return self.model
+
+    @staticmethod
+    def get_recommended_params(
+        task_type: str
+    ) -> Dict[str, Any]:
+        """
+        Get recommended verbosity and reasoning_effort for common tasks.
+
+        Args:
+            task_type: One of "translation", "analysis", "routing", "summary"
+
+        Returns:
+            Dict with verbosity and reasoning_effort settings
+        """
+        presets = {
+            "translation": {
+                "verbosity": "medium",
+                "reasoning_effort": "minimal"
+            },
+            "analysis": {
+                "verbosity": "high",
+                "reasoning_effort": "high"
+            },
+            "routing": {
+                "verbosity": "low",
+                "reasoning_effort": "minimal"
+            },
+            "summary": {
+                "verbosity": "medium",
+                "reasoning_effort": "medium"
+            },
+            "comparison": {
+                "verbosity": "high",
+                "reasoning_effort": "high"
+            }
+        }
+        return presets.get(task_type, {"verbosity": "medium", "reasoning_effort": "medium"})
+
+    def _retry_with_backoff(
+        self,
+        fn: Callable[[], Any],
+        *,
+        max_attempts: int = 3,
+        initial_delay: float = 1.0
+    ) -> Any:
+        """
+        Retry function with exponential backoff.
+
+        Retries on transient errors only (timeouts, rate limits).
+        Re-raises immediately on permanent errors (auth, invalid model).
+        """
+        last_error = None
+        delay = initial_delay
+
+        for attempt in range(1, max_attempts + 1):
+            try:
+                return fn()
+
+            except Exception as e:
+                last_error = e
+                error_str = str(e).lower()
+
+                # Don't retry on permanent errors
+                if any(x in error_str for x in [
+                    'api key', 'auth', 'forbidden', 'unauthorized',
+                    'invalid model', 'model not found'
+                ]):
+                    raise LLMError(f"Permanent LLM error: {e}") from e
+
+                # Don't retry on programming errors
+                if isinstance(e, (ValueError, TypeError, KeyError)):
+                    raise LLMError(f"LLM call failed (bad input): {e}") from e
+
+                # Retry on transient errors
+                if attempt < max_attempts:
+                    logger.warning(
+                        "LLM call failed (attempt %d/%d): %s. Retrying in %.1fs...",
+                        attempt, max_attempts, e, delay
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+                    continue
+
+        # All retries exhausted
+        raise LLMError(
+            f"LLM call failed after {max_attempts} attempts: {last_error}"
+        ) from last_error
+
+    def chat(
+        self,
+        messages: List[Dict[str, str]],
+        *,
+        max_tokens: int = 900,
+        temperature: float = 0.2,
+        top_p: float = 1.0,
+        stream: bool = False,
+        model_type: Optional[ModelType] = None,
+        verbosity: Optional[VerbosityLevel] = None,
+        reasoning_effort: Optional[ReasoningEffort] = None,
+    ) -> str:
+        """
+        Single chat completion call with optional streaming.
+
+        Args:
+            messages: List of message dicts with 'role' and 'content'
+            max_tokens: Maximum tokens to generate
+            temperature: Sampling temperature (0-2)
+            top_p: Nucleus sampling parameter
+            stream: If True, returns generator yielding tokens (else full response)
+            model_type: Type of model to use ('router', 'translator', or 'analyzer')
+            verbosity: Response length control (GPT-5 feature)
+                - 'low': Brief, concise responses
+                - 'medium': Standard length responses
+                - 'high': Detailed, comprehensive responses
+            reasoning_effort: Thinking time control (GPT-5 feature)
+                - 'minimal': Quick, straightforward responses
+                - 'medium': Moderate analysis and reasoning
+                - 'high': Deep analysis and careful reasoning
+
+        Recommended combinations:
+            - Translations: verbosity='medium', reasoning_effort='minimal'
+            - Complex analysis: verbosity='high', reasoning_effort='high'
+            - Routing decisions: verbosity='low', reasoning_effort='minimal'
+
+        Returns:
+            Generated text response (or generator if stream=True)
+
+        Raises:
+            LLMError: On API failures or invalid responses
+            ConfigError: If client not initialized
+        """
+        if not self.is_ready():
+            if self.disable_llm:
+                return self._fallback_response(messages)
+            raise LLMError(
+                "LLM client not initialized. Check API key and configuration."
+            )
+
+        # Select model based on type
+        selected_model = self._get_model_for_type(model_type)
+
+        def _make_call():
+            try:
+                # Build API call parameters
+                params = {
+                    "model": selected_model,
+                    "messages": messages,
+                    "temperature": temperature,
+                    "top_p": top_p,
+                    "max_tokens": max_tokens,
+                    "stream": stream,
+                }
+
+                # Add GPT-5 specific parameters if provided
+                if verbosity is not None:
+                    params["verbosity"] = verbosity
+                if reasoning_effort is not None:
+                    params["reasoning_effort"] = reasoning_effort
+
+                resp = self.client.chat.completions.create(**params)
+            except Exception as e:
+                # Handle httpx exceptions if available
+                if httpx and isinstance(e, httpx.TimeoutException):
+                    raise LLMError(f"LLM request timed out") from e
+                elif httpx and isinstance(e, httpx.HTTPStatusError):
+                    status = e.response.status_code
+                    raise LLMError(
+                        f"LLM API error (HTTP {status}): {e.response.text[:200]}"
+                    ) from e
+                else:
+                    raise LLMError(f"LLM API call failed: {e}") from e
+
+            if stream:
+                # Return generator for streaming
+                return self._stream_response(resp)
+            else:
+                # Validate response
+                if not resp.choices:
+                    raise LLMError("LLM returned no choices")
+
+                content = resp.choices[0].message.content
+                if not content:
+                    raise LLMError("LLM returned empty response")
+
+                # Record metrics (token usage and model distribution)
+                try:
+                    from src.monitoring import record_tokens, record_model_use
+                    if hasattr(resp, 'usage') and resp.usage:
+                        record_tokens(
+                            prompt_tokens=resp.usage.prompt_tokens,
+                            completion_tokens=resp.usage.completion_tokens
+                        )
+                    record_model_use(selected_model)
+                except Exception as e:
+                    # Don't fail the request if metrics recording fails
+                    logging.warning(f"Failed to record metrics: {e}")
+
+                return content.strip()
+
+        return self._retry_with_backoff(_make_call, max_attempts=self.max_retries)
+
+    def _stream_response(self, stream_resp):
+        """
+        Handle streaming response from OpenAI API.
+        Yields chunks of text as they arrive.
+        """
+        try:
+            full_text = ""
+            for chunk in stream_resp:
+                if chunk.choices and chunk.choices[0].delta.content:
+                    token = chunk.choices[0].delta.content
+                    full_text += token
+                    yield token
+        except Exception as e:
+            raise LLMError(f"Stream error: {e}") from e
+
+    def summarize(
+        self,
+        user_text: str,
+        *,
+        system_text: str = "You are an expert UK grant analyst. Be precise, grounded, concise.",
+        max_tokens: int = 900,
+        temperature: float = 0.2,
+        verbosity: Optional[VerbosityLevel] = None,
+        reasoning_effort: Optional[ReasoningEffort] = None,
+    ) -> str:
+        """
+        Convenience: single-turn chat.
+
+        Args:
+            user_text: User message content
+            system_text: System prompt
+            max_tokens: Maximum tokens
+            temperature: Sampling temperature
+            verbosity: Response length control (GPT-5)
+            reasoning_effort: Thinking time control (GPT-5)
+
+        Returns:
+            Generated summary
+
+        Raises:
+            LLMError: On API failures
+        """
+        messages = [
+            {"role": "system", "content": system_text},
+            {"role": "user", "content": user_text},
+        ]
+        return self.chat(
+            messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            verbosity=verbosity,
+            reasoning_effort=reasoning_effort
+        )
+
+    def summarize_long(
+        self,
+        user_text: str,
+        *,
+        system_text: str = "You are an expert UK grant analyst. Be precise, grounded, concise.",
+        max_tokens: int = 1300,
+        temperature: float = 0.2,
+        hard_clip_chars: int = 60_000,
+    ) -> str:
+        """
+        Larger token budget with optional hard clip on input.
+
+        Prevents 400 errors from overly long prompts.
+        """
+        if len(user_text) > hard_clip_chars:
+            logger.warning(
+                "Input text truncated from %d to %d chars",
+                len(user_text), hard_clip_chars
+            )
+            user_text = user_text[:hard_clip_chars] + "\n\n[...truncated...]"
+
+        return self.summarize(
+            user_text,
+            system_text=system_text,
+            max_tokens=max_tokens,
+            temperature=temperature
+        )
+
+    def _fallback_response(self, messages: List[Dict]) -> str:
+        """Return safe fallback when LLM is disabled."""
+        user_msg = next(
+            (m['content'] for m in reversed(messages) if m['role'] == 'user'),
+            ""
+        )
+        return f"[LLM disabled] Your query: {user_msg[:100]}..."

analyzer/logging_setup.py

@@ -0,0 +1,35 @@
+"""
+logging_setup.py — consistent logging config
+"""
+from __future__ import annotations
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+_DEFAULT_FMT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s"
+
+def setup_logging(level: Optional[str] = None, *, to_file: bool = True) -> None:
+    """
+    Initialize root logger once. Level may be 'DEBUG'|'INFO'|'WARNING'|'ERROR'.
+    Falls back to env LOG_LEVEL or INFO.
+    Optionally write to logs/YYYY-MM-DD.log.
+    """
+    if getattr(setup_logging, "_configured", False):
+        return
+
+    lvl = (level or os.getenv("LOG_LEVEL") or "INFO").upper()
+    logging.basicConfig(level=getattr(logging, lvl, logging.INFO), format=_DEFAULT_FMT)
+
+    if to_file:
+        logdir = Path("_out/logs")
+        logdir.mkdir(parents=True, exist_ok=True)
+        logfile = logdir / f"{datetime.now().strftime('%Y-%m-%d')}.log"
+        fh = logging.FileHandler(logfile, encoding="utf-8")
+        fh.setFormatter(logging.Formatter(_DEFAULT_FMT))
+        root = logging.getLogger()
+        root.addHandler(fh)
+        logging.info(f"File logging → {logfile}")
+
+    setup_logging._configured = True  # type: ignore[attr-defined]

analyzer/net/fetcher.py

@@ -0,0 +1,75 @@
+# src/analyzer/net/fetcher.py
+from __future__ import annotations
+import os, time, hashlib
+from typing import Optional, Dict
+import httpx
+from bs4 import BeautifulSoup
+
+CACHE_DIR = os.getenv("LINK_CACHE_DIR", "data/link_cache")
+HEADERS = {"User-Agent": "grant-analyst/1.0 (+polite; contact: engineering@example.com)"}
+TIMEOUT = httpx.Timeout(25.0)
+
+def _sha(s: str) -> str: 
+    return hashlib.sha1(s.encode("utf-8", "ignore")).hexdigest()
+
+def _path(stem: str, ext: str) -> str:
+    os.makedirs(CACHE_DIR, exist_ok=True)
+    return os.path.join(CACHE_DIR, f"{stem}.{ext}")
+
+def _html2text(html: str) -> str:
+    soup = BeautifulSoup(html, "lxml")
+    for sel in ["nav", "header", "footer", ".govuk-footer", ".govuk-phase-banner"]:
+        for n in soup.select(sel):
+            n.decompose()
+    txt = "\n".join(p.get_text(" ", strip=True) for p in soup.find_all(["h1","h2","h3","p","li","dt","dd"]))
+    return txt.strip()
+
+def fetch_link(url: str, *, force: bool = False) -> Dict:
+    """
+    Fetch a URL (HTML or PDF), parse to text, and cache results.
+    Returns: {url, ok, kind, text, cached_txt, cached_html, fetched_at}
+    """
+    key = _sha(url)
+    cached_txt  = _path(key, "txt")
+    cached_html = _path(key, "html")
+    meta_path   = _path(key, "meta")
+
+    if not force and os.path.exists(cached_txt):
+        with open(cached_txt, "r", encoding="utf-8") as f:
+            text = f.read()
+        return {"url": url, "ok": True, "kind": "cached", "text": text,
+                "cached_txt": cached_txt, "cached_html": cached_html,
+                "fetched_at": os.path.getmtime(cached_txt)}
+
+    kind = "html"
+    try:
+        with httpx.Client(timeout=TIMEOUT, follow_redirects=True, headers=HEADERS) as cli:
+            r = cli.get(url)
+            ctype = r.headers.get("content-type","").lower()
+            if "application/pdf" in ctype or url.lower().endswith(".pdf") or ".pdf?" in url.lower():
+                kind = "pdf"
+                pdf_path = _path(key, "pdf")
+                with open(pdf_path, "wb") as f:
+                    f.write(r.content)
+                try:
+                    import fitz  # PyMuPDF
+                    with fitz.open(pdf_path) as doc:
+                        pages = [p.get_text() for p in doc]
+                    text = "\n".join(pages).strip()
+                except Exception:
+                    text = "(PDF saved; text extraction failed)"
+            else:
+                html = r.text
+                text = _html2text(html)
+                with open(cached_html, "w", encoding="utf-8") as f:
+                    f.write(html)
+    except Exception as e:
+        return {"url": url, "ok": False, "error": str(e), "kind": kind}
+
+    with open(cached_txt, "w", encoding="utf-8") as f:
+        f.write(text)
+    with open(meta_path, "w", encoding="utf-8") as f:
+        f.write(f"{time.time()}\n{url}\n{kind}\n")
+    return {"url": url, "ok": True, "kind": kind, "text": text,
+            "cached_txt": cached_txt, "cached_html": cached_html,
+            "fetched_at": time.time()}

analyzer/prompt_templates.py

@@ -0,0 +1,81 @@
+"""
+prompt_templates.py — prompt(s) for grounded + open-style answers
+
+Exports:
+- build_prompt(provider, context_text, style="default") -> dict   (your existing structured summary prompt)
+- build_open_prompt(provider, question, context=None) -> dict     (lighter guardrails; LLM chooses format)
+"""
+
+from __future__ import annotations
+from typing import Dict
+
+# -------------------- Structured (existing) --------------------
+
+_SYSTEM_DEFAULT = (
+    "You are an expert grant analyst. Produce crisp, factual executive summaries "
+    "for UK innovation funding calls using ONLY the provided context. "
+    "For vague or messy user requests, call the tool `search_grants` with `query` set to the raw user text and include any obvious `filters` you can infer (e.g., status=\"open\", audience=\"SME\", theme like \"battery\", or timeframe). "
+    "Be precise, avoid hype, and NEVER invent facts. If a detail is missing, say so briefly. "
+    "Prefer bullet points. Keep to 250–400 words."
+)
+
+_USER_TEMPLATE_DEFAULT = (
+    "CONTEXT (verbatim, may include past winners):\n\n"
+    "{context}\n\n"
+    "TASK: Write an insightful, self-contained summary for an SME reviewer. Use Markdown with these sections:\n\n"
+    "### Summary\n"
+    "One tight paragraph on what this competition is about and who it's for.\n\n"
+    "### Key points\n"
+    "• Funding scope, eligibility, key dates, award magnitude/rate (if present).\n"
+    "• Any geographic/sector focus.\n\n"
+    "### What stands out\n"
+    "• 2–5 observations that would matter to a savvy applicant (novel angles, strict requirements, unusual constraints).\n\n"
+    "### Past winners insight\n"
+    "If past winners are present in the context, briefly note common themes or approaches that succeeded; otherwise say 'No past winners referenced.'\n\n"
+    "### Risks / unknowns\n"
+    "• Bullet any gaps or ambiguities in the call text that an applicant should clarify.\n\n"
+    "### Actionable next steps\n"
+    "• 3–5 concise, practical steps for an SME deciding whether to proceed.\n\n"
+    "Rules: Base everything strictly on the context; do not add external knowledge; do not exceed 400 words."
+)
+
+def build_prompt(provider: str, context_text: str, *, style: str = "default") -> Dict:
+    """Return a provider-appropriate prompt payload for structured summaries."""
+    system = _SYSTEM_DEFAULT
+    user = _USER_TEMPLATE_DEFAULT.format(context=context_text)
+
+    provider = (provider or "openai").lower()
+    if provider == "openai":
+        return {"messages": [{"role": "system", "content": system},
+                             {"role": "user", "content": user}]}
+    elif provider == "anthropic":
+        return {"system": system, "messages": [{"role": "user", "content": user}]}
+    else:
+        return {"messages": [{"role": "system", "content": system},
+                             {"role": "user", "content": user}]}
+
+# -------------------- Open / flexible answering --------------------
+
+OPEN_SYSTEM = (
+    "You are a UK grant analyst and research copilot.\n"
+    "- Prefer grounded answers using the provided context/snippets when available.\n"
+    "- If a detail isn’t in the context, say so briefly or mark it as uncertain.\n"
+    "- Choose the clearest format for the user’s ask (short answer, bullets, table, or brief narrative) — your call.\n"
+    "- Be concise by default; expand only if asked.\n"
+    "- Never fabricate URLs or specific numbers not present in context."
+)
+
+def build_open_prompt(provider: str, question: str, context: str | None = None) -> Dict:
+    """Return a provider-appropriate prompt payload for open, lightly-guarded responses."""
+    provider = (provider or "openai").lower()
+    user = f"QUESTION:\n{question.strip()}\n\nCONTEXT:\n{(context or '').strip()}\n".strip()
+    if provider == "anthropic":
+        return {"system": OPEN_SYSTEM, "messages": [{"role": "user", "content": user}]}
+    return {"messages": [{"role": "system", "content": OPEN_SYSTEM},
+                         {"role": "user", "content": user}]}
+
+# -------------------- Self-test (optional) --------------------
+if __name__ == "__main__":
+    ctx = "TITLE: Example Grant\nDESCRIPTION: Support for AI in manufacturing."
+    print("OpenAI structured:\n", build_prompt("openai", ctx))
+    print("OpenAI open:\n", build_open_prompt("openai", "What is the grant about?", ctx))

analyzer/run_generate.py

@@ -0,0 +1,104 @@
+"""
+run_generate.py — CLI entrypoint for the summarizer (OPTIMIZED)
+
+Loads current grant snapshots and optional past winners, builds an LLM context
+for each grant, retrieves an insightful summary via OpenAI or Anthropic, and
+exports results to Excel/JSONL.
+
+OPTIMIZED VERSION: 14x faster using async parallelization + batching + caching
+
+Usage
+-----
+python -m src.analyzer.run_generate \
+  --snapshots-dir data/snapshots \
+  --history-xlsx data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx \
+  --out-xlsx data/insight_summaries.xlsx \
+  --out-jsonl data/insight_summaries.jsonl \
+  --limit 10 \
+  --include-context
+
+Performance: 30 grants in ~30 seconds (vs 7 minutes before)
+"""
+from __future__ import annotations
+
+import asyncio
+import argparse
+import logging
+import time
+from pathlib import Path
+from typing import Optional
+from dotenv import load_dotenv; load_dotenv()
+
+from .config import load_config
+from .data_loader import load_current_grants, load_past_winners
+from .summarizer_optimized import summarize_grants_async, SummaryCache  # NEW: Optimized version
+from .exporters import export_excel, export_jsonl
+
+
+async def async_main(argv: Optional[list[str]] = None) -> None:
+    """Async version of main for optimized grant summarization."""
+    cfg = load_config()
+
+    logging.basicConfig(
+        level=getattr(logging, cfg.log_level.upper(), logging.INFO),
+        format="%(levelname)s: %(message)s",
+    )
+
+    parser = argparse.ArgumentParser(description="Generate insightful summaries for grant snapshots (OPTIMIZED)")
+    parser.add_argument("--snapshots-dir", type=Path, default=Path("data/snapshots"), help="Directory of current-grant JSON snapshots")
+    parser.add_argument("--history-xlsx", type=Path, default=Path("data/IUK-141025-InnovateUKFundedProjects-FY2015-16topresent.xlsx"), help="Excel file of past winners (optional)")
+    parser.add_argument("--history-json", type=Path, default=None, help="Directory of past winners as JSON files (optional)")
+    parser.add_argument("--out-xlsx", type=Path, default=Path("data/insight_summaries.xlsx"))
+    parser.add_argument("--out-jsonl", type=Path, default=Path("data/insight_summaries.jsonl"))
+    parser.add_argument("--limit", type=int, default=0, help="Process at most N grants (0 = all)")
+    parser.add_argument("--include-context", action="store_true", help="Include the raw context text in the output JSONL/Excel")
+    parser.add_argument("--batch-size", type=int, default=5, help="Grants per API call (default: 5)")
+
+    args = parser.parse_args(argv)
+
+    # 1) Load data
+    current = load_current_grants(args.snapshots_dir, limit=args.limit or None)
+    history = load_past_winners(history_xlsx=args.history_xlsx, history_json_dir=args.history_json)
+
+    if not current:
+        logging.warning("No current grants found under %s", args.snapshots_dir)
+        return
+
+    # 2) Summarize (NEW: Optimized async version with caching)
+    logging.info("Summarizing %d grants (batch_size=%d)...", len(current), args.batch_size)
+    start_time = time.time()
+
+    cache = SummaryCache(ttl_seconds=3600)  # 1-hour TTL cache
+    rows = await summarize_grants_async(
+        current,
+        past_winners=history or None,
+        limit=args.limit or None,
+        include_context=args.include_context,
+        batch_size=args.batch_size,
+        cache=cache,
+    )
+
+    elapsed = time.time() - start_time
+    logging.info("✅ Summarized %d grants in %.1f seconds (%.2f sec/grant)",
+                 len(rows), elapsed, elapsed / len(rows) if rows else 0)
+
+    # Cache stats
+    stats = cache.stats()
+    logging.info("Cache: %d total, %d valid entries", stats["cached"], stats["valid"])
+
+    # 3) Export
+    if args.out_xlsx:
+        export_excel(rows, args.out_xlsx)
+        logging.info("Saved Excel: %s", args.out_xlsx)
+    if args.out_jsonl:
+        export_jsonl(rows, args.out_jsonl)
+        logging.info("Saved JSONL: %s", args.out_jsonl)
+
+
+def main(argv: Optional[list[str]] = None) -> None:
+    """Sync wrapper to run async main."""
+    asyncio.run(async_main(argv))
+
+
+if __name__ == "__main__":
+    main()

analyzer/search/build_index.py

@@ -0,0 +1,80 @@
+# src/analyzer/search/build_index.py
+from __future__ import annotations
+from pathlib import Path
+import argparse, json, pickle, os
+
+from ..data_loader_supporting import load_supporting_docs
+
+def main():
+    ap = argparse.ArgumentParser(description="Build hybrid index from supporting JSON/JSONL")
+    ap.add_argument("--support-dir", type=Path, default=Path("data/supporting/json"),
+                    help="Folder containing competition-*.json or *.jsonl")
+    ap.add_argument("--out-dir", type=Path, default=Path("data/index"),
+                    help="Output folder for index + docs.jsonl")
+    args = ap.parse_args()
+
+    os.makedirs(args.out_dir, exist_ok=True)
+
+    docs = load_supporting_docs(args.support_dir)
+
+    # Helpful logging
+    print(f"Scanning: {args.support_dir.resolve()}")
+    print(f"Found {len(docs)} supporting docs")
+    if not docs:
+        # Show a quick diagnostic to help you place files correctly
+        samples = list(sorted(args.support_dir.glob("*")))[:10]
+        if samples:
+            print("Sample files in folder:")
+            for s in samples:
+                print(" -", s.name)
+        else:
+            print("Folder is empty.")
+    # Write jsonl for inspection
+    jsonl = args.out_dir / "docs.jsonl"
+    with jsonl.open("w", encoding="utf-8") as f:
+        for d in docs:
+            f.write(json.dumps({
+                "id": d.grant_id,
+                "title": d.title,
+                "url": d.url,
+                "open_date": d.open_date,
+                "close_date": d.close_date,
+                "notify_date": d.notify_date,
+                "funding_min": d.funding_min,
+                "funding_max": d.funding_max,
+                "total_pot": d.total_pot,
+                "duration_min": d.duration_min,
+                "duration_max": d.duration_max,
+                "text": d.text[:1000],  # shorten for quick view
+            }, ensure_ascii=False) + "\n")
+
+    # Minimal index payload (ready for BM25/embeddings later)
+    payload = {
+        "version": 1,
+        "docs": [{
+            "id": d.grant_id,
+            "title": d.title,
+            "url": d.url,
+            "meta": {
+                "open_date": d.open_date,
+                "close_date": d.close_date,
+                "notify_date": d.notify_date,
+                "funding_min": d.funding_min,
+                "funding_max": d.funding_max,
+                "total_pot": d.total_pot,
+                "duration_min": d.duration_min,
+                "duration_max": d.duration_max,
+            },
+            "text": d.text
+        } for d in docs]
+    }
+
+    with (args.out_dir / "hybrid_index.pkl").open("wb") as f:
+        pickle.dump(payload, f)
+
+    print(f"✅ Built hybrid index with {len(docs)} supporting docs")
+    print(f"→ {args.out_dir/'hybrid_index.pkl'}")
+    print(f"→ {jsonl}")
+
+if __name__ == "__main__":
+    main()

analyzer/search/hybrid_index.py

@@ -0,0 +1,476 @@
+# src/analyzer/search/hybrid_index.py
+"""
+Consolidated hybrid search index with BM25-style ranking.
+
+Combines grant records + supporting documents into a unified TF-IDF index
+with configurable source prioritization and filtering.
+"""
+from __future__ import annotations
+from dataclasses import dataclass, asdict
+from typing import List, Dict, Tuple, Optional, Protocol
+import os
+import pickle
+import logging
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import linear_kernel
+
+from ..utils.errors import SearchError, DataLoadError
+from ..utils.text import clean
+
+logger = logging.getLogger(__name__)
+
+INDEX_DIR = "data/index"
+INDEX_PATH = os.path.join(INDEX_DIR, "hybrid_index.pkl")
+INDEX_VERSION = 2  # Increment when schema changes
+
+
+# ---------------------------------------------------------------------------
+# Data Models
+# ---------------------------------------------------------------------------
+
+@dataclass
+class IndexedDoc:
+    """Represents a searchable document (grant or supporting material)."""
+    id: str
+    title: str
+    deadline: str
+    url: str
+    _source: str          # "grant" | "supporting"
+    section: str          # "" for grants, section name for supporting
+    competition_id: str
+    text: str
+    meta: Dict = None     # Additional metadata (funding, dates, etc.)
+
+    def __post_init__(self):
+        if self.meta is None:
+            self.meta = {}
+
+
+# ---------------------------------------------------------------------------
+# Index Building
+# ---------------------------------------------------------------------------
+
+# Text normalization moved to utils.text.clean()
+# Keeping wrapper for backward compatibility
+def _normalize_text(s: str) -> str:
+    """Clean text for indexing."""
+    return clean(s)
+
+
+def _normalize_grant_id(gid: str) -> str:
+    """Normalize grant/competition ID by removing prefixes."""
+    if not gid:
+        return ""
+    gid = str(gid).strip()
+    # Remove common prefixes to ensure consistency
+    for prefix in ["competition-", "grant-"]:
+        if gid.startswith(prefix):
+            gid = gid[len(prefix):]
+    return gid
+
+
+def build_index(
+    grants: List[Dict], 
+    supporting_docs: List[Dict],
+    *,
+    ngram_range: Tuple[int, int] = (1, 2),
+    max_df: float = 0.95,
+    min_df: int = 2
+) -> Dict:
+    """
+    Build a hybrid TF-IDF index from grants and supporting documents.
+    
+    Args:
+        grants: List of grant dicts (from data_loader)
+        supporting_docs: List of supporting doc dicts
+        ngram_range: N-gram range for TF-IDF (default: unigrams + bigrams)
+        max_df: Ignore terms in >X% of docs (remove very common words)
+        min_df: Ignore terms in <X docs (remove rare words)
+    
+    Returns:
+        Dict with keys: docs, X (TF-IDF matrix), vectorizer, metadata
+    """
+    docs: List[IndexedDoc] = []
+    texts: List[str] = []
+
+    # Index grants
+    for g in grants:
+        gid = _normalize_grant_id(g.get("id") or g.get("competition_id") or "")
+        if not gid:
+            continue
+
+        title = _normalize_text(g.get("title", ""))
+        parts = [
+            title,
+            _normalize_text(g.get("summary", "")),
+            _normalize_text(g.get("overview", "")),
+            _normalize_text(g.get("scope", "")),
+            _normalize_text(g.get("eligibility", ""))
+        ]
+        text = "\n".join([p for p in parts if p])
+
+        docs.append(IndexedDoc(
+            id=gid,
+            title=title or f"Grant {gid}",
+            deadline=_normalize_text(g.get("deadline", "")),
+            url=_normalize_text(g.get("url") or g.get("source_url") or ""),
+            _source="grant",
+            section="",
+            competition_id=gid,
+            text=text or title or gid,
+            meta={
+                "funding_min": g.get("funding", {}).get("min"),
+                "funding_max": g.get("funding", {}).get("max"),
+                "open_date": g.get("open_date"),
+                "close_date": g.get("close_date")
+            }
+        ))
+        texts.append(text or title or gid)
+
+    # Index supporting documents
+    for s in supporting_docs:
+        gid = _normalize_grant_id(s.get("grant_id") or s.get("competition_id") or "")
+        if not gid:
+            continue
+
+        sec = _normalize_text(s.get("section", "Supporting"))
+        ttl = _normalize_text(s.get("title", ""))
+        body = _normalize_text(s.get("text", ""))
+        url = _normalize_text(s.get("url") or s.get("source_url") or "")
+
+        blob = f"[{sec}] {ttl}\n{body}" if ttl else f"[{sec}]\n{body}"
+
+        docs.append(IndexedDoc(
+            id=f"{gid}::support::{sec[:60]}",
+            title=ttl or f"{sec} — {gid}",
+            deadline="",
+            url=url,
+            _source="supporting",
+            section=sec,
+            competition_id=gid,
+            text=blob or sec or ttl or gid,
+            meta={"parent_grant_id": gid}
+        ))
+        texts.append(blob or sec or ttl or gid)
+
+    if not docs:
+        raise DataLoadError(
+            "No documents to index. "
+            "Provide at least one grant or supporting document."
+        )
+
+    # Build TF-IDF matrix
+    try:
+        logger.info(f"Building index with {len(docs)} docs (ngram_range={ngram_range})")
+        vectorizer = TfidfVectorizer(
+            lowercase=True,
+            ngram_range=ngram_range,
+            max_df=max_df,
+            min_df=min_df,
+            strip_accents='unicode'
+        )
+        X = vectorizer.fit_transform(texts)
+        logger.info(f"Index built: {X.shape[0]} docs, {X.shape[1]} features")
+    except Exception as e:
+        raise SearchError(f"Failed to build TF-IDF index: {e}") from e
+    
+    return {
+        "docs": docs,
+        "X": X,
+        "vectorizer": vectorizer,
+        "metadata": {
+            "version": INDEX_VERSION,
+            "n_grants": sum(1 for d in docs if d._source == "grant"),
+            "n_supporting": sum(1 for d in docs if d._source == "supporting"),
+            "n_features": X.shape[1]
+        }
+    }
+
+
+# ---------------------------------------------------------------------------
+# Index Persistence
+# ---------------------------------------------------------------------------
+
+def save_index(idx: Dict, path: str = INDEX_PATH) -> None:
+    """Save index to disk with versioning."""
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    
+    # Convert IndexedDoc instances to dicts for pickling
+    serializable_docs = [asdict(doc) for doc in idx["docs"]]
+    
+    payload = {
+        "docs": serializable_docs,
+        "X": idx["X"],
+        "vectorizer": idx["vectorizer"],
+        "metadata": idx.get("metadata", {})
+    }
+    
+    with open(path, "wb") as f:
+        pickle.dump(payload, f, protocol=pickle.HIGHEST_PROTOCOL)
+    
+    logger.info(f"Index saved to {path}")
+
+
+def load_index(path: str = INDEX_PATH) -> Dict:
+    """
+    Load index from disk.
+
+    Args:
+        path: Path to index pickle file
+
+    Returns:
+        Index dictionary
+
+    Raises:
+        DataLoadError: If file not found or corrupt
+        SearchError: If index version incompatible or structure invalid
+    """
+    if not os.path.exists(path):
+        raise DataLoadError(
+            f"Index not found: {path}. "
+            f"Build it first with: python -m src.analyzer.search.hybrid_index"
+        )
+
+    try:
+        with open(path, "rb") as f:
+            payload = pickle.load(f)
+    except Exception as e:
+        raise DataLoadError(
+            f"Failed to load index from {path}: {e}. "
+            f"Try rebuilding the index."
+        ) from e
+
+    # Validate structure
+    required_keys = {"docs", "X", "vectorizer"}
+    missing = required_keys - set(payload.keys())
+    if missing:
+        raise SearchError(
+            f"Index file corrupt: missing keys {missing}. "
+            f"Rebuild the index."
+        )
+
+    # Reconstruct IndexedDoc instances
+    try:
+        docs = [IndexedDoc(**d) for d in payload["docs"]]
+        payload["docs"] = docs
+    except Exception as e:
+        raise SearchError(
+            f"Failed to reconstruct documents from index: {e}"
+        ) from e
+
+    # Version check
+    version = payload.get("metadata", {}).get("version", 1)
+    if version < INDEX_VERSION:
+        logger.warning(
+            "Index version %d is outdated (current: %d). "
+            "Consider rebuilding for best results.",
+            version, INDEX_VERSION
+        )
+
+    logger.info("Index loaded: %d docs", len(docs))
+    return payload
+
+
+# ---------------------------------------------------------------------------
+# Search
+# ---------------------------------------------------------------------------
+
+def _compute_source_prior(docs: List[IndexedDoc]) -> np.ndarray:
+    """
+    Apply source-based ranking boost.
+    Grants get 1.1x boost, supporting docs get 1.0x.
+    """
+    return np.array([1.10 if d._source == "grant" else 1.00 for d in docs], dtype=np.float32)
+
+
+def search(
+    idx: Dict,
+    query: str,
+    *,
+    k: int = 10,
+    filters: Optional[Dict] = None
+) -> List[Tuple[Dict, float]]:
+    """
+    Search the index with TF-IDF similarity + source boosting.
+    
+    Args:
+        idx: Index dict from build_index() or load_index()
+        query: Search query string
+        k: Number of results to return
+        filters: Optional dict with keys:
+            - competition_id: str (filter to specific grant)
+            - source: "grant" | "supporting" (filter by doc type)
+            - min_score: float (minimum similarity threshold)
+    
+    Returns:
+        List of (doc_dict, score) tuples, sorted by score descending
+    """
+    if not query or not query.strip():
+        return []
+    
+    filters = filters or {}
+    vectorizer = idx["vectorizer"]
+    X = idx["X"]
+    docs: List[IndexedDoc] = idx["docs"]
+    
+    # Compute query vector and similarities
+    query_vec = vectorizer.transform([query])
+    similarities = linear_kernel(query_vec, X).ravel().astype(np.float32)
+    
+    # Apply source prior boost
+    similarities *= _compute_source_prior(docs)
+    
+    # Apply filters
+    mask = np.ones(len(docs), dtype=bool)
+
+    if "competition_id" in filters and filters["competition_id"]:
+        cid = _normalize_grant_id(filters["competition_id"])
+        mask &= np.array([d.competition_id == cid for d in docs], dtype=bool)
+    
+    if "source" in filters and filters["source"]:
+        src = str(filters["source"])
+        mask &= np.array([d._source == src for d in docs], dtype=bool)
+    
+    if "min_score" in filters and filters["min_score"]:
+        min_score = float(filters["min_score"])
+        mask &= (similarities >= min_score)
+    
+    # Zero out masked scores
+    similarities[~mask] = -1e9
+    
+    # Get top-k
+    if k >= len(similarities):
+        top_indices = np.argsort(-similarities)
+    else:
+        top_indices = np.argpartition(-similarities, kth=k)[:k]
+        top_indices = top_indices[np.argsort(-similarities[top_indices])]
+    
+    # Build results
+    results = []
+    for idx_pos in top_indices:
+        score = float(similarities[idx_pos])
+        if score <= 0:  # Skip masked/irrelevant docs
+            continue
+        results.append((asdict(docs[idx_pos]), score))
+    
+    return results
+
+
+def search_by_grant_id(idx: Dict, grant_id: str, k: int = 5) -> List[Tuple[Dict, float]]:
+    """
+    Retrieve documents for a specific grant (useful for enrichment).
+
+    Returns grant doc + supporting docs sorted by relevance to a generic query.
+    If no query matches, returns all docs for the grant sorted by source (grant first).
+    """
+    # Normalize the grant_id to match index format
+    normalized_id = _normalize_grant_id(grant_id)
+    docs: List[IndexedDoc] = idx["docs"]
+
+    # Filter to just this grant's docs
+    matching_docs = [
+        (i, doc) for i, doc in enumerate(docs)
+        if doc.competition_id == normalized_id
+    ]
+
+    if not matching_docs:
+        return []
+
+    # Try search with query first
+    results = search(
+        idx,
+        query="supporting information guidance terms eligibility scope",
+        k=k,
+        filters={"competition_id": normalized_id}
+    )
+
+    # If search returns results, use those
+    if results:
+        return results
+
+    # Otherwise, return all matching docs sorted by source (grants first)
+    # with dummy scores based on source priority
+    sorted_docs = sorted(
+        matching_docs,
+        key=lambda x: (0 if x[1]._source == "grant" else 1, x[0])
+    )
+
+    return [
+        (asdict(doc), 1.0 if doc._source == "grant" else 0.5)
+        for _, doc in sorted_docs[:k]
+    ]
+
+
+# ---------------------------------------------------------------------------
+# Convenience Functions
+# ---------------------------------------------------------------------------
+
+def top_supporting_for_grant(idx: Dict, grant_id: str, *, k: int = 6) -> List[Tuple[Dict, float]]:
+    """
+    Alias for search_by_grant_id - maintains backward compatibility.
+    """
+    return search_by_grant_id(idx, grant_id, k=k)
+
+
+def rebuild_index_from_data(
+    snapshots_dir: str = "data/snapshots",
+    supporting_dir: str = "data/supporting/json",
+    output_path: str = INDEX_PATH
+) -> Dict:
+    """
+    Convenience function to rebuild index from scratch.
+    
+    Usage:
+        from analyzer.search.hybrid_index import rebuild_index_from_data
+        idx = rebuild_index_from_data()
+    """
+    from pathlib import Path
+    from ..data_loader import load_current_grants
+    from ..data_loader_supporting import load_supporting_docs
+    
+    grants = load_current_grants(Path(snapshots_dir))
+    supporting = load_supporting_docs(Path(supporting_dir))
+    
+    # Convert SupportingDoc dataclass to dict if needed
+    supporting_dicts = []
+    for s in supporting:
+        if hasattr(s, '__dict__'):
+            supporting_dicts.append({
+                'grant_id': s.grant_id,
+                'section': s.title,
+                'title': s.title,
+                'text': s.text,
+                'url': s.url,
+            })
+        else:
+            supporting_dicts.append(s)
+    
+    idx = build_index(grants, supporting_dicts)
+    save_index(idx, output_path)
+    
+    return idx
+
+
+# ---------------------------------------------------------------------------
+# Self-Test
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import logging
+    logging.basicConfig(level=logging.INFO)
+    
+    # Example: rebuild index
+    try:
+        idx = rebuild_index_from_data()
+        print(f"✅ Index built successfully")
+        print(f"   Metadata: {idx['metadata']}")
+        
+        # Test search
+        results = search(idx, "AI battery research", k=3)
+        print(f"\n🔍 Test search for 'AI battery research':")
+        for doc, score in results[:3]:
+            print(f"   [{score:.3f}] {doc['title']}")
+    
+    except Exception as e:
+        print(f"❌ Error: {e}")

analyzer/search/past_winners_integration.py

@@ -0,0 +1,181 @@
+"""
+Previous Winners Integration Module
+
+Integrates historical winners data into the search index for:
+- "Who won [Grant]?" queries
+- Past winner recommendations/case studies
+- Success pattern analysis
+"""
+
+from typing import Dict, List, Any, Optional
+from pathlib import Path
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def convert_past_winner_to_searchable(winner: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Convert a past winner record to searchable format for hybrid index.
+
+    Args:
+        winner: Past winner dict with fields like project_title, competition, etc.
+
+    Returns:
+        Dict suitable for indexing as a supporting document
+    """
+    return {
+        "grant_id": winner.get("competition", ""),
+        "section": f"Past Winner: {winner.get('project_title', 'Unknown')}",
+        "title": winner.get("project_title", "Unknown Project"),
+        "text": " ".join([
+            f"Project: {winner.get('project_title', '')}",
+            f"Organization: {winner.get('lead_org', '')}",
+            f"Year: {winner.get('year', '')}",
+            f"Award: £{winner.get('award_amount', '')}",
+            f"Summary: {winner.get('abstract', '')}",
+        ]),
+        "url": winner.get("project_url", ""),
+        "meta": {
+            "source": "past_winner",
+            "year": winner.get("year"),
+            "award_amount": winner.get("award_amount"),
+            "lead_org": winner.get("lead_org"),
+        }
+    }
+
+
+def enrich_index_with_past_winners(
+    idx: Dict[str, Any],
+    past_winners: List[Dict[str, Any]]
+) -> Dict[str, Any]:
+    """
+    Enrich an existing hybrid index with past winner documents.
+
+    Note: This is in-memory enrichment. For persistence, rebuild the full index.
+
+    Args:
+        idx: Existing hybrid index dict
+        past_winners: List of past winner records
+
+    Returns:
+        Modified index dict with past winner documents added
+    """
+    if not past_winners:
+        logger.info("No past winners to integrate")
+        return idx
+
+    from .hybrid_index import IndexedDoc
+
+    # Convert past winners to searchable documents
+    searchable_winners = [convert_past_winner_to_searchable(w) for w in past_winners]
+
+    logger.info(f"Integrating {len(searchable_winners)} past winner records")
+
+    # Add to docs list
+    for winner_doc in searchable_winners:
+        grant_id = winner_doc.get("grant_id", "").lower().replace("competition-", "")
+
+        indexed_doc = IndexedDoc(
+            id=f"{grant_id}::winner::{winner_doc.get('title', 'unknown')}",
+            title=winner_doc.get("title", "Past Winner"),
+            deadline="",
+            url=winner_doc.get("url", ""),
+            _source="past_winner",
+            section=winner_doc.get("section", ""),
+            competition_id=grant_id,
+            text=winner_doc.get("text", ""),
+            meta=winner_doc.get("meta", {})
+        )
+
+        idx["docs"].append(indexed_doc)
+
+    logger.info(f"Index now contains {len(idx['docs'])} documents (with past winners)")
+    return idx
+
+
+def format_past_winner_summary(winner: Dict[str, Any]) -> str:
+    """
+    Format a past winner record as readable markdown.
+
+    Args:
+        winner: Past winner dict
+
+    Returns:
+        Markdown formatted summary
+    """
+    lines = [
+        f"### {winner.get('project_title', 'Unknown Project')}",
+        "",
+        f"**Organization:** {winner.get('lead_org', 'Unknown')}",
+        f"**Award:** £{winner.get('award_amount', 'Unknown'):,.0f}" if isinstance(
+            winner.get('award_amount'), (int, float)
+        ) else f"**Award:** {winner.get('award_amount', 'Unknown')}",
+        f"**Year:** {winner.get('year', 'Unknown')}",
+        "",
+        f"**Description:**",
+        f"{winner.get('abstract', 'No description available')}",
+        "",
+    ]
+
+    if winner.get("project_url"):
+        lines.append(f"**Project Link:** {winner['project_url']}")
+
+    return "\n".join(lines)
+
+
+def find_past_winners_for_grant(
+    idx: Dict[str, Any],
+    grant_id: str,
+    limit: int = 5
+) -> List[Dict[str, Any]]:
+    """
+    Find past winners for a specific grant.
+
+    Args:
+        idx: Hybrid index dict
+        grant_id: Grant ID to find winners for
+        limit: Max winners to return
+
+    Returns:
+        List of past winner documents
+    """
+    from .hybrid_index import _normalize_grant_id
+
+    normalized_id = _normalize_grant_id(grant_id)
+    winners = []
+
+    for doc in idx.get("docs", []):
+        if (doc.get("_source") == "past_winner" and
+                doc.get("competition_id") == normalized_id):
+            winners.append(doc)
+
+    return winners[:limit]
+
+
+def search_past_winners(
+    idx: Dict[str, Any],
+    query: str,
+    limit: int = 10
+) -> List[Dict[str, Any]]:
+    """
+    Search past winners by project name or organization.
+
+    Args:
+        idx: Hybrid index dict
+        query: Search query
+        limit: Max results
+
+    Returns:
+        List of matching past winner documents
+    """
+    from .hybrid_index import search
+
+    results = search(
+        idx,
+        query=query,
+        k=limit,
+        filters={"source": "past_winner"}
+    )
+
+    return [doc for doc, score in results]

analyzer/search/query.py

@@ -0,0 +1,32 @@
+# src/analyzer/search/query.py
+"""
+DEPRECATED: This module has been consolidated into hybrid_index.py
+
+For migration, use:
+  - HybridIndex -> load_index() from hybrid_index
+  - index.search() -> search(index, query, k=5)
+  - index.by_grant_id() -> search_by_grant_id(index, grant_id)
+"""
+import warnings
+warnings.warn(
+    "query.py is deprecated. Use hybrid_index.py instead.",
+    DeprecationWarning,
+    stacklevel=2
+)
+
+# Keep old class as a thin wrapper for backward compatibility
+from .hybrid_index import load_index, search, search_by_grant_id
+
+class HybridIndex:
+    """Deprecated wrapper. Use functions from hybrid_index directly."""
+    def __init__(self, path):
+        warnings.warn("HybridIndex class is deprecated", DeprecationWarning)
+        self._idx = load_index(str(path))
+
+    def search(self, query: str, limit: int = 5):
+        results = search(self._idx, query, k=limit)
+        return [doc for doc, score in results]
+
+    def by_grant_id(self, gid: str):
+        results = search_by_grant_id(self._idx, gid, k=10)
+        return [doc for doc, score in results]

analyzer/streaming_summarizer.py

@@ -0,0 +1,267 @@
+"""
+streaming_summarizer.py — Advanced streaming + parallel grant summarization
+
+Implements:
+1. True parallelization with configurable concurrency
+2. OpenAI streaming (stream=True) for real-time token delivery
+3. Per-grant streaming with immediate feedback
+4. Graceful fallback for Gradio (not async context)
+"""
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional, AsyncGenerator
+
+from .llm_client import LLMClient
+from .summarizer_optimized import SummaryCache, extract_minimal_context
+
+logger = logging.getLogger(__name__)
+
+
+async def summarize_grant_streaming(
+    grant: Dict[str, Any],
+    client: LLMClient,
+    cache: SummaryCache,
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+) -> AsyncGenerator[str, None]:
+    """
+    Stream a single grant summary token-by-token.
+
+    Yields tokens as they arrive from OpenAI.
+    """
+    grant_id = grant.get("id") or grant.get("title") or "unknown"
+    title = grant.get("title") or grant.get("name") or "(untitled)"
+
+    # Check cache first
+    cached_summary = cache.get(grant)
+    if cached_summary:
+        logger.info("📦 Cache HIT for %s", grant_id)
+        # Stream cached content quickly
+        for token in cached_summary.split():
+            yield token + " "
+        return
+
+    # Build context
+    context = extract_minimal_context(grant, past_winners)
+
+    # Stream from OpenAI
+    from .prompt_templates import build_prompt
+
+    payload = build_prompt("openai", context)
+    full_response = ""
+
+    try:
+        # Use stream=True to get token-by-token delivery
+        stream = client.chat(payload["messages"], max_tokens=1200, temperature=0.25, stream=True)
+
+        for token in stream:
+            full_response += token
+            yield token
+
+        # Cache the full response
+        cache.set(grant, full_response)
+        logger.info("✅ Cached summary for %s", grant_id)
+
+    except Exception as e:
+        logger.error("❌ Stream failed for %s: %s", grant_id, e)
+        error_msg = f"Error generating summary: {str(e)[:100]}"
+        cache.set(grant, error_msg)
+        yield error_msg
+
+
+async def summarize_grants_parallel_streaming(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    client: Optional[LLMClient] = None,
+    cache: Optional[SummaryCache] = None,
+    batch_size: int = 5,
+    max_concurrent: int = 3,
+) -> AsyncGenerator[Dict[str, Any], None]:
+    """
+    Parallel + Streaming: Process multiple grants concurrently.
+
+    Each grant streams its own summary as it's being generated.
+    Results yielded as they complete (not in order).
+
+    Args:
+        max_concurrent: Max grants to process simultaneously (default: 3)
+    """
+    client = client or LLMClient({})
+    cache = cache or SummaryCache(ttl_seconds=3600)
+
+    items = current[: limit or len(current)]
+
+    if not items:
+        return
+
+    # Create a semaphore to limit concurrent tasks
+    semaphore = asyncio.Semaphore(max_concurrent)
+
+    async def bounded_summarize(grant, index):
+        """Summarize with concurrency limit."""
+        async with semaphore:
+            grant_id = grant.get("id") or grant.get("title") or "unknown"
+            title = grant.get("title") or grant.get("name") or "(untitled)"
+
+            full_summary = ""
+
+            try:
+                logger.info(f"[{index+1}/{len(items)}] Summarizing: {title[:50]}")
+
+                async for token in summarize_grant_streaming(grant, client, cache, past_winners):
+                    full_summary += token
+
+                return {
+                    "grant_id": grant_id,
+                    "title": title,
+                    "summary_md": full_summary,
+                    "index": index,
+                }
+            except Exception as e:
+                logger.error(f"Failed to summarize {grant_id}: {e}")
+                return {
+                    "grant_id": grant_id,
+                    "title": title,
+                    "summary_md": f"Error: {str(e)[:100]}",
+                    "index": index,
+                }
+
+    # Create all tasks
+    tasks = [bounded_summarize(grant, i) for i, grant in enumerate(items)]
+
+    # Yield results as they complete (using as_completed)
+    for coro in asyncio.as_completed(tasks):
+        result = await coro
+        yield result
+
+    logger.info("✅ Completed all %d grants", len(items))
+
+
+async def summarize_grants_batch_parallel_streaming(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    client: Optional[LLMClient] = None,
+    cache: Optional[SummaryCache] = None,
+    batch_size: int = 5,
+    max_concurrent_batches: int = 2,
+) -> AsyncGenerator[Dict[str, Any], None]:
+    """
+    Optimized: Process batches in parallel, stream batch results.
+
+    5 grants per batch → reduced API calls
+    Multiple batches in parallel → maximum throughput
+    Results stream as soon as batch completes
+
+    This is the recommended approach for 30+ grants.
+    """
+    from .summarizer_optimized import _summarize_batch_async
+
+    client = client or LLMClient({})
+    cache = cache or SummaryCache(ttl_seconds=3600)
+
+    items = current[: limit or len(current)]
+
+    if not items:
+        return
+
+    # Build contexts
+    contexts = [extract_minimal_context(g, past_winners) for g in items]
+
+    # Create batches
+    batches = [
+        (items[i:i+batch_size], contexts[i:i+batch_size])
+        for i in range(0, len(items), batch_size)
+    ]
+
+    # Create a semaphore for concurrent batch processing
+    semaphore = asyncio.Semaphore(max_concurrent_batches)
+
+    async def process_batch(batch_items, batch_contexts, batch_idx):
+        """Process a batch with concurrency limit."""
+        async with semaphore:
+            try:
+                logger.info(f"Processing batch {batch_idx+1}/{len(batches)} ({len(batch_items)} grants)")
+                batch_results = await _summarize_batch_async(batch_items, batch_contexts, client, cache)
+
+                for result in batch_results:
+                    yield result
+
+            except Exception as e:
+                logger.error(f"Batch {batch_idx} failed: {e}")
+                for grant in batch_items:
+                    yield {
+                        "grant_id": grant.get("id") or "unknown",
+                        "title": grant.get("title") or "(untitled)",
+                        "summary_md": f"Batch error: {str(e)[:100]}",
+                    }
+
+    # Process batches concurrently
+    tasks = [
+        process_batch(batch_items, batch_contexts, i)
+        for i, (batch_items, batch_contexts) in enumerate(batches)
+    ]
+
+    # Yield from all tasks as they complete
+    for task in asyncio.as_completed(tasks):
+        async for result in task:
+            yield result
+
+    logger.info("✅ All batches processed")
+
+
+# Synchronous wrapper for use in non-async contexts (e.g., Gradio callbacks)
+def summarize_grants_streaming_sync(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    client: Optional[LLMClient] = None,
+    cache: Optional[SummaryCache] = None,
+    batch_size: int = 5,
+    mode: str = "batch_parallel",  # "grant_parallel" or "batch_parallel"
+) -> List[Dict[str, Any]]:
+    """
+    Synchronous wrapper for streaming summarization.
+
+    Returns all results as a list (blocking until complete).
+    Use this when you need results in order.
+    """
+    async def run():
+        results = []
+        if mode == "batch_parallel":
+            async for result in summarize_grants_batch_parallel_streaming(
+                current,
+                past_winners=past_winners,
+                limit=limit,
+                client=client,
+                cache=cache,
+                batch_size=batch_size,
+            ):
+                results.append(result)
+        else:  # grant_parallel
+            async for result in summarize_grants_parallel_streaming(
+                current,
+                past_winners=past_winners,
+                limit=limit,
+                client=client,
+                cache=cache,
+                batch_size=batch_size,
+            ):
+                results.append(result)
+
+        # Sort by original index if available
+        return sorted(results, key=lambda x: x.get("index", float('inf')))
+
+    try:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # Already in async context, return async generator
+            raise RuntimeError("Use async version directly in async context")
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+    return loop.run_until_complete(run())

analyzer/summarizer.py

@@ -0,0 +1,107 @@
+"""
+summarizer.py — glue logic: load → build context → call LLM → collect results
+
+Public API
+----------
+- summarize_grants(current: list[dict], past_winners: list[dict] | None = None,
+                    *, limit: int | None = None, include_context: bool = False) -> list[dict]
+
+Returns a list of dicts with keys:
+  grant_id, title, summary_md, context(optional), source_path(optional)
+
+This file intentionally stays light; exporting and CLI live elsewhere.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+from pathlib import Path
+import logging
+
+from .context_builder import build_context
+from .llm_client import LLMClient
+
+logger = logging.getLogger(__name__)
+
+# ----------------------------- Core API ---------------------------------------
+
+def summarize_grants(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    include_context: bool = False,
+    client: Optional[LLMClient] = None,
+) -> List[Dict[str, Any]]:
+    """Summarize a batch of grants using an LLM.
+
+    Parameters
+    ----------
+    current : list of grant dicts (from data_loader.load_current_grants)
+    past_winners : optional list of past winner dicts (may be empty)
+    limit : if provided, process at most this many grants
+    include_context : whether to include the raw context text in the result
+    client : optional pre-initialized LLMClient
+    """
+    client = client or LLMClient()
+
+    # For this MVP, we pass the *same* past_winners list to every grant.
+    # Later you can add filtering by theme if you want.
+    results: List[Dict[str, Any]] = []
+
+    items = current[: limit or len(current)]
+    for i, g in enumerate(items, 1):
+        grant_id = g.get("id") or g.get("title") or g.get("name") or f"grant_{i}"
+        title = g.get("title") or g.get("name") or g.get("competition_title") or "(untitled)"
+        try:
+            ctx = build_context(g, past_winners)
+            summary = client.summarize(ctx)
+            row = {
+                "grant_id": grant_id,
+                "title": title,
+                "summary_md": summary,
+            }
+            if include_context:
+                row["context"] = ctx
+            if g.get("_path"):
+                row["source_path"] = g["_path"]
+            results.append(row)
+            logger.info("Summarized: %s", title)
+        except Exception as e:  # keep going even if one fails
+            logger.exception("Failed to summarize %s: %s", title, e)
+            results.append({
+                "grant_id": grant_id,
+                "title": title,
+                "summary_md": f"Summary failed: {e}",
+            })
+    return results
+
+
+# ----------------------------- Ad-hoc test ------------------------------------
+if __name__ == "__main__":
+    # Tiny smoke test using fake data
+    logging.basicConfig(level=logging.INFO)
+    fake_current = [
+        {
+            "id": "demo-1",
+            "title": "AI in Manufacturing",
+            "sections": {
+                "summary_raw": "Funding for AI-driven manufacturing improvements.",
+                "scope_raw": "Projects should demonstrate measurable productivity gains.",
+            },
+            "deadline": "2025-12-17",
+            "funding_amount": "up to £1M",
+        }
+    ]
+    fake_history = [
+        {
+            "project_title": "Smart Factory Vision",
+            "lead_org": "Acme Robotics",
+            "award_amount": "£450,000",
+            "competition": "Manufacturing AI 2023",
+            "abstract": "Computer vision for automated QA on production lines.",
+        }
+    ]
+
+    out = summarize_grants(fake_current, fake_history, limit=1, include_context=True)
+    from pprint import pprint
+    pprint(out)

analyzer/summarizer_optimized.py

@@ -0,0 +1,590 @@
+"""
+summarizer_optimized.py — High-performance grant summarization with:
+- PARALLELIZATION: asyncio.gather() for concurrent processing
+- STREAMING: Yield results as batches complete (for async contexts)
+- BATCH PROCESSING: 5 grants per API call via clever prompting
+- SMART CONTEXT: Extract only essential fields (~200 tokens per grant)
+- CACHING: In-memory cache with 1-hour TTL (Redis-ready pattern)
+- MODEL OPTIMIZATION: gpt-5-mini for basic summaries (fast and cost-effective)
+
+Performance target: <30 seconds for 30 grants (vs. 7 minutes sequential)
+
+Public API
+----------
+- summarize_grants_optimized(current, past_winners=None, limit=None, include_context=False,
+                             client=None, cache=None, stream=False, batch_size=5)
+  -> List[Dict] or AsyncGenerator[Dict] (if stream=True)
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import logging
+import time
+from typing import Any, Dict, List, Optional, AsyncGenerator, Tuple
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from .context_builder import build_context
+from .llm_client import LLMClient
+
+logger = logging.getLogger(__name__)
+
+# Import tiktoken for token counting
+try:
+    import tiktoken
+    HAS_TIKTOKEN = True
+except ImportError:
+    HAS_TIKTOKEN = False
+    logger.warning("tiktoken not available - token counting disabled")
+
+
+# ================================= CACHING LAYER =================================
+
+class SummaryCache:
+    """Simple in-memory cache with TTL support. Redis-ready: replace dict with redis.Redis."""
+
+    def __init__(self, ttl_seconds: int = 3600):
+        self.cache: Dict[str, Tuple[str, float]] = {}  # hash -> (summary, timestamp)
+        self.ttl_seconds = ttl_seconds
+
+    def _hash_grant(self, grant: Dict[str, Any]) -> str:
+        """Create deterministic hash of grant ID and essential fields."""
+        key_parts = [
+            grant.get("id", ""),
+            grant.get("title", ""),
+            grant.get("deadline", ""),
+        ]
+        key_str = "|".join(str(p) for p in key_parts)
+        return hashlib.md5(key_str.encode()).hexdigest()
+
+    def get(self, grant: Dict[str, Any]) -> Optional[str]:
+        """Retrieve cached summary if exists and not expired."""
+        h = self._hash_grant(grant)
+        if h in self.cache:
+            summary, timestamp = self.cache[h]
+            if time.time() - timestamp < self.ttl_seconds:
+                logger.debug("Cache HIT for %s", grant.get("title", "unknown"))
+                return summary
+            else:
+                del self.cache[h]  # Expired
+        return None
+
+    def set(self, grant: Dict[str, Any], summary: str) -> None:
+        """Store summary in cache."""
+        h = self._hash_grant(grant)
+        self.cache[h] = (summary, time.time())
+        logger.debug("Cache SET for %s", grant.get("title", "unknown"))
+
+    def stats(self) -> Dict[str, int]:
+        """Return cache statistics."""
+        now = time.time()
+        valid = sum(1 for _, (_, ts) in self.cache.items() if now - ts < self.ttl_seconds)
+        return {"cached": len(self.cache), "valid": valid}
+
+
+# ================================= CONTEXT EXTRACTION =================================
+
+def _get_first_sentences(text: str, n: int = 3) -> str:
+    """Extract first N sentences from text."""
+    if not text:
+        return ""
+    sentences = text.split('. ')
+    return '. '.join(sentences[:n]).strip() + ('.' if len(sentences) > n else '')
+
+
+def _count_tokens(text: str) -> int:
+    """Count tokens in text using tiktoken (if available)."""
+    if not HAS_TIKTOKEN:
+        # Rough approximation: 1 token ≈ 4 characters
+        return len(text) // 4
+
+    try:
+        # Use o200k_base encoding for GPT-5 models (fallback to cl100k_base for GPT-4)
+        try:
+            encoding = tiktoken.get_encoding("o200k_base")
+        except:
+            encoding = tiktoken.get_encoding("cl100k_base")
+        return len(encoding.encode(text))
+    except Exception:
+        # Fallback to character approximation
+        return len(text) // 4
+
+
+def extract_minimal_context(grant: Dict[str, Any], past_winners: Optional[List[Dict[str, Any]]] = None) -> str:
+    """
+    Extract only essential fields (~200 tokens per grant).
+
+    Aggressive reduction strategy:
+    - Title: max 100 chars
+    - Deadline: as-is
+    - Funding: as-is
+    - Summary: first 3 sentences only
+    - Eligibility: first 2 sentences only
+    - NO past winners (saves ~50-100 tokens)
+
+    Target: <200 tokens per grant for efficient batch processing
+    """
+    parts = []
+
+    # Title (truncate to 100 chars)
+    title = grant.get("title") or grant.get("name") or grant.get("competition_title") or "(untitled)"
+    title = title[:100]
+    parts.append(f"TITLE: {title}")
+
+    # Deadline
+    deadline = grant.get("deadline") or grant.get("close_date")
+    if deadline:
+        parts.append(f"DEADLINE: {deadline}")
+
+    # Funding
+    funding = grant.get("funding_amount") or grant.get("max_funding")
+    if funding:
+        parts.append(f"FUNDING: {funding}")
+
+    # Summary/Description (first 3 sentences only)
+    summary_raw = None
+    for field in ["summary_raw", "summary", "description", "overview"]:
+        if grant.get("sections", {}).get(field):
+            summary_raw = grant["sections"][field]
+            break
+
+    if summary_raw:
+        # Extract first 3 sentences
+        summary_short = _get_first_sentences(summary_raw, 3)
+        # Further truncate to 200 chars if needed
+        if len(summary_short) > 200:
+            summary_short = summary_short[:200] + "..."
+        parts.append(f"SUMMARY: {summary_short}")
+
+    # Eligibility (first 2 sentences only)
+    eligibility = None
+    for field in ["eligibility_raw", "eligibility", "who_can_apply"]:
+        if grant.get("sections", {}).get(field):
+            eligibility = grant["sections"][field]
+            break
+
+    if eligibility:
+        # Extract first 2 sentences
+        eligibility_short = _get_first_sentences(eligibility, 2)
+        # Further truncate to 150 chars if needed
+        if len(eligibility_short) > 150:
+            eligibility_short = eligibility_short[:150] + "..."
+        parts.append(f"ELIGIBILITY: {eligibility_short}")
+
+    # Build final context
+    context = "\n".join(parts)
+
+    # Token count check (for logging)
+    token_count = _count_tokens(context)
+    if token_count > 200:
+        logger.debug(
+            f"Context for '{title[:30]}...' is {token_count} tokens (target: 200)"
+        )
+
+    return context
+
+
+# ================================= BATCH SUMMARIZATION =================================
+
+def _build_batch_prompt(grants_batch: List[Dict[str, Any]], contexts: List[str]) -> str:
+    """
+    Build a single prompt for summarizing multiple grants.
+
+    Returns plain text summaries rather than JSON to avoid parsing issues.
+    """
+    prompt_parts = [
+        f"Summarize these {len(grants_batch)} grants. For EACH grant, provide a DETAILED summary (150-250 words).\n",
+        f"Use this format for each grant:\n",
+        f"### Grant [NUMBER]: [GRANT TITLE]\n",
+        f"[DETAILED SUMMARY]\n\n",
+    ]
+
+    for i, (grant, ctx) in enumerate(zip(grants_batch, contexts), 1):
+        prompt_parts.append(f"\n--- GRANT {i} ---")
+        prompt_parts.append(ctx)
+
+    return "\n".join(prompt_parts)
+
+
+async def _summarize_batch_async(
+    grants_batch: List[Dict[str, Any]],
+    contexts: List[str],
+    client: LLMClient,
+    cache: SummaryCache,
+) -> List[Dict[str, Any]]:
+    """
+    Summarize a batch of grants in a single API call.
+    Returns list of {grant_id, title, summary_md} dicts.
+    """
+    results = []
+
+    # Check cache first
+    cached_grants = []
+    uncached_grants = []
+    uncached_indices = []
+
+    for idx, (grant, ctx) in enumerate(zip(grants_batch, contexts)):
+        cached_summary = cache.get(grant)
+        if cached_summary:
+            results.append({
+                "grant_id": grant.get("id") or grant.get("title") or f"grant_{idx}",
+                "title": grant.get("title") or grant.get("name") or "(untitled)",
+                "summary_md": cached_summary,
+            })
+        else:
+            uncached_grants.append(grant)
+            uncached_indices.append(idx)
+
+    if not uncached_grants:
+        return results
+
+    # Batch summarize uncached grants
+    try:
+        batch_prompt = _build_batch_prompt(uncached_grants, [contexts[i] for i in uncached_indices])
+
+        # Use faster model for batch summaries
+        system_prompt = (
+            "You are an expert UK grant analyst. Provide DETAILED, THOROUGH summaries for each grant. "
+            "Use markdown formatting. Be comprehensive and informative."
+        )
+
+        # Run in thread pool to avoid blocking
+        loop = asyncio.get_event_loop()
+        summary_text = await loop.run_in_executor(
+            None,
+            lambda: client.summarize(
+                batch_prompt,
+                system_text=system_prompt,
+                max_tokens=4000,  # Increased to allow detailed summaries (200-250 words per grant)
+            )
+        )
+
+        # Parse plain text response (no JSON)
+        try:
+            summary_text = summary_text.strip()
+            # Split by "### Grant" to get individual summaries
+            import re
+            grant_sections = re.split(r'###\s+Grant\s+\d+:', summary_text)
+
+            summaries = []
+            for i, section in enumerate(grant_sections[1:], 1):  # Skip first empty split
+                # Extract grant title and summary
+                lines = section.strip().split('\n')
+                if lines:
+                    summary = '\n'.join(lines).strip()
+                    if summary:
+                        summaries.append(summary)
+
+            # If we didn't get enough summaries, fill with empty ones
+            while len(summaries) < len(uncached_grants):
+                summaries.append("[Summary generation failed]")
+
+        except Exception as e:
+            logger.warning("Failed to parse batch response: %s", e)
+            # Fallback: return empty summaries
+            summaries = ["[Summary generation failed]" for _ in uncached_grants]
+
+        # Map summaries back to original grants
+        for orig_idx, (grant, summary) in enumerate(zip(uncached_grants, summaries)):
+            grant_id = grant.get("id") or grant.get("title") or f"grant_{orig_idx}"
+            title = grant.get("title") or grant.get("name") or "(untitled)"
+
+            result = {
+                "grant_id": grant_id,
+                "title": title,
+                "summary_md": summary,
+            }
+            results.append(result)
+
+            # Cache the summary
+            cache.set(grant, summary)
+            logger.info("Summarized (batch): %s", title)
+
+        return results
+
+    except Exception as e:
+        logger.exception("Batch summarization failed: %s", e)
+        # Fallback: return error messages
+        for grant in uncached_grants:
+            results.append({
+                "grant_id": grant.get("id") or grant.get("title") or "unknown",
+                "title": grant.get("title") or grant.get("name") or "(untitled)",
+                "summary_md": f"Summary failed: {str(e)[:100]}",
+            })
+        return results
+
+
+# ================================= ASYNC ORCHESTRATION =================================
+
+async def summarize_grants_async(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    include_context: bool = False,
+    client: Optional[LLMClient] = None,
+    cache: Optional[SummaryCache] = None,
+    batch_size: int = 5,
+) -> List[Dict[str, Any]]:
+    """
+    Async version: Summarize grants in parallel batches.
+
+    Parameters
+    ----------
+    current : list of grant dicts
+    past_winners : optional list of past winner dicts
+    limit : if provided, process at most this many grants
+    include_context : whether to include raw context in result
+    client : optional pre-initialized LLMClient
+    cache : optional SummaryCache instance
+    batch_size : number of grants per API call (default: 5)
+
+    Returns
+    -------
+    List of dicts with keys: grant_id, title, summary_md, context(optional), source_path(optional)
+    """
+    client = client or LLMClient({})
+    cache = cache or SummaryCache(ttl_seconds=3600)
+
+    items = current[: limit or len(current)]
+
+    if not items:
+        return []
+
+    # Build minimal contexts
+    contexts = [extract_minimal_context(g, past_winners) for g in items]
+
+    # Create batches
+    batches = [
+        (items[i:i+batch_size], contexts[i:i+batch_size])
+        for i in range(0, len(items), batch_size)
+    ]
+
+    # Process batches in parallel
+    start_time = time.time()
+    batch_results = await asyncio.gather(
+        *[
+            _summarize_batch_async(batch_items, batch_contexts, client, cache)
+            for batch_items, batch_contexts in batches
+        ],
+        return_exceptions=True
+    )
+
+    elapsed = time.time() - start_time
+    logger.info("Summarized %d grants in %.1f seconds (%.2f sec/grant)",
+                len(items), elapsed, elapsed / len(items) if items else 0)
+
+    # Flatten results
+    results = []
+    for batch_result in batch_results:
+        if isinstance(batch_result, Exception):
+            logger.error("Batch failed: %s", batch_result)
+        else:
+            results.extend(batch_result)
+
+    # Add optional fields
+    for result, grant in zip(results, items[:len(results)]):
+        if include_context:
+            result["context"] = extract_minimal_context(grant, past_winners)
+        if grant.get("_path"):
+            result["source_path"] = grant["_path"]
+
+    # Log cache stats
+    cache_stats = cache.stats()
+    logger.info("Cache stats: %d total, %d valid entries",
+                cache_stats["cached"], cache_stats["valid"])
+
+    return results
+
+
+async def summarize_grants_streaming(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    include_context: bool = False,
+    client: Optional[LLMClient] = None,
+    cache: Optional[SummaryCache] = None,
+    batch_size: int = 5,
+) -> AsyncGenerator[Dict[str, Any], None]:
+    """
+    Async generator: Yield results as each batch completes (streaming).
+
+    Allows UI to display summaries in real-time.
+    """
+    client = client or LLMClient({})
+    cache = cache or SummaryCache(ttl_seconds=3600)
+
+    items = current[: limit or len(current)]
+
+    if not items:
+        return
+
+    # Build minimal contexts
+    contexts = [extract_minimal_context(g, past_winners) for g in items]
+
+    # Create batches
+    batches = [
+        (items[i:i+batch_size], contexts[i:i+batch_size])
+        for i in range(0, len(items), batch_size)
+    ]
+
+    # Process and yield as batches complete
+    for batch_items, batch_contexts in batches:
+        try:
+            batch_results = await _summarize_batch_async(batch_items, batch_contexts, client, cache)
+
+            for result, grant in zip(batch_results, batch_items):
+                if include_context:
+                    result["context"] = extract_minimal_context(grant, past_winners)
+                if grant.get("_path"):
+                    result["source_path"] = grant["_path"]
+
+                yield result
+
+        except Exception as e:
+            logger.error("Batch streaming failed: %s", e)
+            for grant in batch_items:
+                yield {
+                    "grant_id": grant.get("id") or grant.get("title") or "unknown",
+                    "title": grant.get("title") or grant.get("name") or "(untitled)",
+                    "summary_md": f"Summary failed: {str(e)[:100]}",
+                }
+
+
+# ================================= BACKWARD COMPATIBILITY =================================
+
+def summarize_grants(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    *,
+    limit: Optional[int] = None,
+    include_context: bool = False,
+    client: Optional[LLMClient] = None,
+) -> List[Dict[str, Any]]:
+    """
+    DEPRECATED: Use summarize_grants_optimized() instead.
+
+    Synchronous wrapper around async implementation for backward compatibility.
+    """
+    cache = SummaryCache(ttl_seconds=3600)
+
+    # Run async version in event loop
+    try:
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # Already in async context; use synchronous fallback
+            logger.warning("summarize_grants called from async context; performance will be degraded")
+            return _summarize_grants_sync(current, past_winners, limit, include_context, client)
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+    return loop.run_until_complete(
+        summarize_grants_async(current, past_winners, limit=limit,
+                              include_context=include_context, client=client, cache=cache)
+    )
+
+
+def _summarize_grants_sync(
+    current: List[Dict[str, Any]],
+    past_winners: Optional[List[Dict[str, Any]]] = None,
+    limit: Optional[int] = None,
+    include_context: bool = False,
+    client: Optional[LLMClient] = None,
+) -> List[Dict[str, Any]]:
+    """
+    Fallback synchronous implementation (less efficient).
+    Process grants sequentially with batch API calls.
+    """
+    from concurrent.futures import ThreadPoolExecutor
+
+    client = client or LLMClient({})
+    cache = SummaryCache(ttl_seconds=3600)
+
+    items = current[: limit or len(current)]
+    contexts = [extract_minimal_context(g, past_winners) for g in items]
+
+    batches = [
+        (items[i:i+5], contexts[i:i+5])
+        for i in range(0, len(items), 5)
+    ]
+
+    results = []
+
+    def process_batch(batch_items, batch_contexts):
+        return asyncio.run(
+            _summarize_batch_async(batch_items, batch_contexts, client, cache)
+        )
+
+    # Process batches in thread pool
+    with ThreadPoolExecutor(max_workers=3) as executor:
+        batch_results = list(executor.map(
+            lambda args: process_batch(args[0], args[1]),
+            batches
+        ))
+
+    # Flatten
+    for batch_result in batch_results:
+        results.extend(batch_result)
+
+    # Add optional fields
+    for result, grant in zip(results, items[:len(results)]):
+        if include_context:
+            result["context"] = extract_minimal_context(grant, past_winners)
+        if grant.get("_path"):
+            result["source_path"] = grant["_path"]
+
+    return results
+
+
+# ================================= CONVENIENCE ALIAS =================================
+
+summarize_grants_optimized = summarize_grants_async  # Main recommended API
+
+
+# ================================= SMOKE TEST =================================
+
+if __name__ == "__main__":
+    import asyncio
+    logging.basicConfig(level=logging.INFO)
+
+    fake_current = [
+        {
+            "id": "demo-1",
+            "title": "AI in Manufacturing",
+            "sections": {
+                "summary_raw": "Funding for AI-driven manufacturing improvements. " * 50,
+                "eligibility_raw": "Open to SMEs and large enterprises.",
+            },
+            "deadline": "2025-12-17",
+            "funding_amount": "up to £1M",
+        },
+        {
+            "id": "demo-2",
+            "title": "Green Energy Innovation",
+            "sections": {
+                "summary_raw": "Support for renewable energy projects. " * 50,
+                "eligibility_raw": "Academic institutions and non-profits.",
+            },
+            "deadline": "2025-11-30",
+            "funding_amount": "£500k-£2M",
+        },
+    ]
+
+    # Test cache
+    cache = SummaryCache(ttl_seconds=3600)
+    print("Testing cache...")
+    cache.set(fake_current[0], "Test summary")
+    assert cache.get(fake_current[0]) == "Test summary"
+    print("Cache works!")
+
+    # Test context extraction
+    print("\nTesting context extraction...")
+    ctx = extract_minimal_context(fake_current[0])
+    print(f"Context length: {len(ctx)} chars")
+    print(ctx[:200])
+
+    print("\nSmoke tests passed!")

analyzer/telemetry/__init__.py

@@ -0,0 +1,2 @@
+from .logger import QALogger, QATurn
+__all__ = ["QALogger", "QATurn"]

analyzer/telemetry/logger.py

@@ -0,0 +1,54 @@
+"""
+telemetry/logger.py — append-only JSONL logging for Q&A turns.
+
+Usage:
+    from analyzer.telemetry.logger import QALogger
+    log = QALogger("logs/chat.jsonl")
+    log.write(user="find farming grants", intent="search", args={"keyword":"farming"},
+              answer_md="...", ok=True, latency_ms=321, meta={"model":"gpt-5-mini"})
+"""
+from __future__ import annotations
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+import json, os, time, threading, uuid, datetime as dt
+
+@dataclass
+class QATurn:
+    ts: str
+    session_id: str
+    turn_id: str
+    user: str
+    intent: str
+    args: Dict[str, Any] = field(default_factory=dict)
+    answer_md: str = ""
+    ok: bool = True
+    latency_ms: Optional[int] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+class QALogger:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._session = os.getenv("CHAT_SESSION_ID") or str(uuid.uuid4())
+
+    def write(self, *, user: str, intent: str, args: Dict[str, Any],
+              answer_md: str, ok: bool, latency_ms: Optional[int],
+              meta: Optional[Dict[str, Any]] = None) -> None:
+        turn = QATurn(
+            ts=dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
+            session_id=self._session,
+            turn_id=str(uuid.uuid4()),
+            user=str(user),
+            intent=str(intent),
+            args=args or {},
+            answer_md=str(answer_md or ""),
+            ok=bool(ok),
+            latency_ms=int(latency_ms) if latency_ms is not None else None,
+            meta=meta or {},
+        )
+        line = json.dumps(asdict(turn), ensure_ascii=False)
+        with self._lock:
+            with open(self.path, "a", encoding="utf-8") as fh:
+                fh.write(line + "\n")

analyzer/utils/citations.py

@@ -0,0 +1,182 @@
+"""
+Citation and sourcing utilities to reduce hallucinations.
+
+When presenting grant information, we cite the source data:
+- Grant ID and title
+- Specific fields (funding_max, deadline, etc.)
+- Field source (from grant record or extracted)
+
+This helps users verify information and trust the system.
+"""
+
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+
+
+def cite_grant_fact(grant: Dict[str, Any], field: str, value: Any) -> str:
+    """
+    Create a citation for a fact about a grant.
+
+    Args:
+        grant: The grant record dict
+        field: Field name (e.g., "funding_max", "close_date")
+        value: The value to cite
+
+    Returns:
+        Formatted citation like "£50,000 [Source: Grant ID #2315]"
+    """
+    grant_id = grant.get("id") or grant.get("competition_id") or "unknown"
+
+    # Special formatting for currency
+    if field in ("funding_max", "funding_min", "total_pot") and isinstance(value, (int, float)):
+        return f"£{value:,.0f} [Source: Grant ID #{grant_id}]"
+
+    # Special formatting for dates
+    if field in ("close_date", "open_date", "deadline") and value:
+        return f"{value} [Source: Grant ID #{grant_id}]"
+
+    # Default formatting
+    if value is None:
+        return "Not specified"
+
+    return f"{value} [Source: Grant ID #{grant_id}]"
+
+
+def format_grant_with_citations(grant: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Format a grant record with citations on key fields.
+
+    Args:
+        grant: The grant record
+
+    Returns:
+        Dict with cited versions of key fields
+    """
+    grant_id = grant.get("id") or grant.get("competition_id") or "unknown"
+    title = grant.get("title", "(untitled)")
+
+    cited = {
+        "id": grant_id,
+        "title": title,
+        "url": grant.get("url", ""),
+    }
+
+    # Cite monetary fields
+    if grant.get("funding_max"):
+        cited["funding_max"] = cite_grant_fact(grant, "funding_max", grant["funding_max"])
+    if grant.get("funding_min"):
+        cited["funding_min"] = cite_grant_fact(grant, "funding_min", grant["funding_min"])
+    if grant.get("total_pot"):
+        cited["total_pot"] = cite_grant_fact(grant, "total_pot", grant["total_pot"])
+
+    # Cite dates
+    if grant.get("close_date"):
+        cited["close_date"] = cite_grant_fact(grant, "close_date", grant["close_date"])
+    if grant.get("open_date"):
+        cited["open_date"] = cite_grant_fact(grant, "open_date", grant["open_date"])
+
+    return cited
+
+
+def build_citation_summary(grant: Dict[str, Any]) -> str:
+    """
+    Build a markdown summary of grant with citations.
+
+    Args:
+        grant: The grant record
+
+    Returns:
+        Markdown formatted summary with citations
+    """
+    grant_id = grant.get("id") or grant.get("competition_id") or "unknown"
+    title = grant.get("title", "(untitled)")
+    url = grant.get("url", "")
+
+    lines = [
+        f"## {title}",
+        f"**Grant ID:** #{grant_id}",
+        "",
+    ]
+
+    if url:
+        lines.append(f"**Official Link:** {url}")
+        lines.append("")
+
+    # Funding details with citations
+    if grant.get("funding_max") or grant.get("funding_min"):
+        min_fund = grant.get("funding_min")
+        max_fund = grant.get("funding_max")
+
+        if min_fund and max_fund:
+            funding_str = f"£{min_fund:,.0f} – £{max_fund:,.0f}"
+        elif max_fund:
+            funding_str = f"up to £{max_fund:,.0f}"
+        elif min_fund:
+            funding_str = f"from £{min_fund:,.0f}"
+        else:
+            funding_str = "See official page"
+
+        lines.append(f"**Funding per project:** {funding_str} [Source: Grant ID #{grant_id}]")
+
+    if grant.get("total_pot"):
+        lines.append(f"**Total available:** £{grant['total_pot']:,.0f} [Source: Grant ID #{grant_id}]")
+
+    lines.append("")
+
+    # Dates with citations
+    if grant.get("open_date") or grant.get("close_date"):
+        close = grant.get("close_date") or grant.get("deadline")
+        if close:
+            lines.append(f"**Deadline:** {close} [Source: Grant ID #{grant_id}]")
+        if grant.get("open_date"):
+            lines.append(f"**Opens:** {grant['open_date']} [Source: Grant ID #{grant_id}]")
+
+    lines.append("")
+
+    # Duration with citation
+    if grant.get("duration_min") or grant.get("duration_max"):
+        min_dur = grant.get("duration_min")
+        max_dur = grant.get("duration_max")
+
+        if min_dur and max_dur:
+            duration_str = f"{min_dur}–{max_dur} months"
+        elif max_dur:
+            duration_str = f"up to {max_dur} months"
+        elif min_dur:
+            duration_str = f"from {min_dur} months"
+        else:
+            duration_str = "Variable"
+
+        lines.append(f"**Project duration:** {duration_str} [Source: Grant ID #{grant_id}]")
+
+    return "\n".join(lines)
+
+
+def validate_fact_availability(grant: Dict[str, Any], fact_type: str) -> bool:
+    """
+    Check if a fact exists in the grant data before citing it.
+
+    Args:
+        grant: The grant record
+        fact_type: Type of fact ("funding", "deadline", "duration", "scope", "eligibility")
+
+    Returns:
+        True if the fact is available to cite
+
+    Raises assertion/warning if fact is missing
+    """
+    fact_fields = {
+        "funding": ["funding_max", "funding_min", "total_pot"],
+        "deadline": ["close_date", "deadline"],
+        "duration": ["duration_min", "duration_max"],
+        "scope": ["scope", "scope_raw", "sections"],
+        "eligibility": ["eligibility", "eligibility_raw"],
+    }
+
+    required_fields = fact_fields.get(fact_type, [])
+
+    for field in required_fields:
+        if grant.get(field):
+            return True
+
+    return False

analyzer/utils/dates.py

@@ -0,0 +1,88 @@
+"""
+utils/dates.py — Date parsing and formatting utilities
+
+Functions:
+- parse_date(s): Parse common date formats to datetime
+- format_date(s): Format date to readable string
+"""
+from __future__ import annotations
+from datetime import datetime
+from typing import Any, Optional
+
+
+def parse_date(s: Any) -> Optional[datetime]:
+    """
+    Parse common date formats to datetime object.
+
+    Supports:
+    - ISO 8601: 2024-12-31T23:59:59
+    - Date only: 2024-12-31
+    - UK format: 31/12/2024
+
+    Args:
+        s: Date string or datetime object
+
+    Returns:
+        datetime object or None if parsing fails
+    """
+    if not s:
+        return None
+
+    if isinstance(s, datetime):
+        return s
+
+    # Try common formats
+    s_str = str(s)
+
+    # ISO 8601 with time: 2024-12-31T23:59:59
+    if 'T' in s_str and len(s_str) >= 19:
+        try:
+            return datetime.strptime(s_str[:19], "%Y-%m-%dT%H:%M:%S")
+        except Exception:
+            pass
+
+    # ISO 8601 date only: 2024-12-31
+    if len(s_str) >= 10 and s_str[4:5] == '-' and s_str[7:8] == '-':
+        try:
+            return datetime.strptime(s_str[:10], "%Y-%m-%d")
+        except Exception:
+            pass
+
+    # UK format: 31/12/2024
+    if '/' in s_str and len(s_str) >= 10:
+        try:
+            return datetime.strptime(s_str[:10], "%d/%m/%Y")
+        except Exception:
+            pass
+
+    return None
+
+
+def format_date(s: Any, *, include_time: bool = False) -> str:
+    """
+    Format date to human-readable string.
+
+    Args:
+        s: Date string, datetime object, or None
+        include_time: If True, include time if available
+
+    Returns:
+        Formatted date string or "—" if parsing fails
+
+    Examples:
+        >>> format_date("2024-12-31")
+        '2024-12-31'
+        >>> format_date("2024-12-31T14:30:00", include_time=True)
+        '2024-12-31 14:30:00'
+    """
+    d = parse_date(s)
+    if not d:
+        return str(s) if s else "—"
+
+    # Auto-detect if time should be included
+    has_time = d.hour or d.minute or d.second
+
+    if include_time or has_time:
+        return d.strftime("%Y-%m-%d %H:%M:%S")
+    else:
+        return d.strftime("%Y-%m-%d")

analyzer/utils/errors.py

@@ -0,0 +1,96 @@
+"""
+Standardized exceptions for the Grant Analyzer system.
+
+Exception Hierarchy:
+    GrantAnalyzerError (base)
+    ├── DataLoadError          # File I/O, parsing
+    ├── ValidationError        # Input validation
+    ├── SearchError            # Index/search issues
+    ├── LLMError              # API/LLM failures
+    └── ConfigError           # Configuration problems
+"""
+
+class GrantAnalyzerError(Exception):
+    """
+    Base exception for all analyzer errors.
+
+    All custom exceptions inherit from this so callers can catch
+    all analyzer-specific errors with a single except clause.
+    """
+    pass
+
+
+class DataLoadError(GrantAnalyzerError):
+    """
+    Error loading or parsing grant/supporting data.
+
+    Examples:
+        - JSON file not found
+        - Malformed JSON
+        - Missing required fields
+        - Excel read failure
+    """
+    pass
+
+
+class ValidationError(GrantAnalyzerError):
+    """
+    User input validation failure.
+
+    Examples:
+        - Invalid grant ID format
+        - Empty search query
+        - URL not in allowlist
+        - Negative funding amount
+    """
+    pass
+
+
+class SearchError(GrantAnalyzerError):
+    """
+    Search or index operation failure.
+
+    Examples:
+        - Index file corrupt
+        - Index version mismatch
+        - Search timeout
+        - Too many results
+    """
+    pass
+
+
+class LLMError(GrantAnalyzerError):
+    """
+    LLM API call failure.
+
+    Examples:
+        - API key invalid
+        - Rate limit exceeded
+        - Timeout
+        - Model not found
+        - Response parsing error
+    """
+    pass
+
+
+class ConfigError(GrantAnalyzerError):
+    """
+    Configuration error.
+
+    Examples:
+        - Missing required env var
+        - Invalid provider name
+        - Conflicting settings
+    """
+    pass
+
+
+# Convenience re-exports
+__all__ = [
+    "GrantAnalyzerError",
+    "DataLoadError",
+    "ValidationError",
+    "SearchError",
+    "LLMError",
+    "ConfigError",
+]

analyzer/utils/query_logger.py

@@ -0,0 +1,317 @@
+"""
+query_logger.py — Logs user queries and AI responses for RLHF analysis
+
+Automatically logs every interaction to CSV and JSONL formats.
+Export to RLHF training format with export_logs.py script.
+"""
+from __future__ import annotations
+
+import csv
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class QueryLogger:
+    """Logs user queries and AI responses to JSON and CSV for RLHF analysis."""
+
+    def __init__(self, log_dir: str = "logs"):
+        """
+        Initialize query logger.
+
+        Args:
+            log_dir: Directory to store log files (default: "logs/")
+        """
+        self.log_dir = Path(log_dir)
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+
+        # Get today's date for log file naming
+        today = datetime.now().strftime("%Y%m%d")
+        self.jsonl_path = self.log_dir / f"queries_{today}.jsonl"
+        self.csv_path = self.log_dir / f"queries_{today}.csv"
+
+        # Initialize CSV file with headers if it doesn't exist
+        if not self.csv_path.exists():
+            self._init_csv()
+
+    def _init_csv(self):
+        """Initialize CSV file with headers."""
+        headers = [
+            'timestamp',
+            'user_query',
+            'ai_response',
+            'tools_called',
+            'response_time_ms',
+            'success',
+            'rating',
+            'feedback',
+            'model',
+            'tokens_used'
+        ]
+        with open(self.csv_path, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.DictWriter(f, fieldnames=headers)
+            writer.writeheader()
+
+    def log_interaction(
+        self,
+        user_query: str,
+        ai_response: str,
+        *,
+        tools_called: Optional[List[str]] = None,
+        response_time_ms: Optional[int] = None,
+        success: bool = True,
+        rating: Optional[int] = None,
+        feedback: Optional[str] = None,
+        model: Optional[str] = None,
+        tokens_used: Optional[int] = None,
+        metadata: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Log a user query and AI response.
+
+        Args:
+            user_query: The user's input query
+            ai_response: The AI's response
+            tools_called: List of tool names that were called
+            response_time_ms: Response time in milliseconds
+            success: Whether the interaction was successful
+            rating: Optional 1-5 rating (for RLHF)
+            feedback: Optional text feedback (for RLHF)
+            model: Model name used (e.g., "gpt-5-mini")
+            tokens_used: Number of tokens consumed
+            metadata: Additional metadata to log
+        """
+        timestamp = datetime.utcnow().isoformat()
+
+        # Prepare log entry
+        log_entry = {
+            'timestamp': timestamp,
+            'user_query': user_query,
+            'ai_response': ai_response,
+            'tools_called': tools_called or [],
+            'response_time_ms': response_time_ms,
+            'success': success,
+            'rating': rating,
+            'feedback': feedback,
+            'model': model,
+            'tokens_used': tokens_used,
+            'metadata': metadata or {}
+        }
+
+        # Write to JSONL (one JSON object per line)
+        try:
+            with open(self.jsonl_path, 'a', encoding='utf-8') as f:
+                f.write(json.dumps(log_entry) + '\n')
+        except Exception as e:
+            logger.warning(f"Failed to write to JSONL log: {e}")
+
+        # Write to CSV (Excel-compatible)
+        try:
+            with open(self.csv_path, 'a', newline='', encoding='utf-8') as f:
+                writer = csv.DictWriter(f, fieldnames=[
+                    'timestamp', 'user_query', 'ai_response', 'tools_called',
+                    'response_time_ms', 'success', 'rating', 'feedback',
+                    'model', 'tokens_used'
+                ])
+                writer.writerow({
+                    'timestamp': timestamp,
+                    'user_query': user_query,
+                    'ai_response': ai_response,
+                    'tools_called': ','.join(tools_called or []),
+                    'response_time_ms': response_time_ms,
+                    'success': success,
+                    'rating': rating or '',
+                    'feedback': feedback or '',
+                    'model': model or '',
+                    'tokens_used': tokens_used or ''
+                })
+        except Exception as e:
+            logger.warning(f"Failed to write to CSV log: {e}")
+
+        logger.debug(f"Logged interaction: {user_query[:50]}...")
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get statistics about logged queries.
+
+        Returns:
+            Dictionary with statistics (total queries, success rate, avg response time, etc.)
+        """
+        try:
+            with open(self.csv_path, 'r', encoding='utf-8') as f:
+                reader = csv.DictReader(f)
+                rows = list(reader)
+
+            if not rows:
+                return {
+                    'total_queries': 0,
+                    'successful_queries': 0,
+                    'failed_queries': 0,
+                    'avg_response_time_ms': 0,
+                    'avg_rating': 0,
+                    'rated_queries': 0,
+                    'tool_usage': {}
+                }
+
+            total = len(rows)
+            successful = sum(1 for r in rows if r.get('success') == 'True')
+            failed = total - successful
+
+            # Calculate average response time
+            response_times = [
+                int(r['response_time_ms'])
+                for r in rows
+                if r.get('response_time_ms') and r['response_time_ms'].isdigit()
+            ]
+            avg_response_time = sum(response_times) / len(response_times) if response_times else 0
+
+            # Calculate average rating
+            ratings = [
+                int(r['rating'])
+                for r in rows
+                if r.get('rating') and r['rating'].isdigit()
+            ]
+            avg_rating = sum(ratings) / len(ratings) if ratings else 0
+            rated_queries = len(ratings)
+
+            # Count tool usage
+            tool_usage = {}
+            for row in rows:
+                tools = row.get('tools_called', '').split(',')
+                for tool in tools:
+                    tool = tool.strip()
+                    if tool:
+                        tool_usage[tool] = tool_usage.get(tool, 0) + 1
+
+            return {
+                'total_queries': total,
+                'successful_queries': successful,
+                'failed_queries': failed,
+                'avg_response_time_ms': avg_response_time,
+                'avg_rating': avg_rating,
+                'rated_queries': rated_queries,
+                'tool_usage': tool_usage
+            }
+        except Exception as e:
+            logger.warning(f"Failed to calculate stats: {e}")
+            return {
+                'total_queries': 0,
+                'successful_queries': 0,
+                'failed_queries': 0,
+                'avg_response_time_ms': 0,
+                'avg_rating': 0,
+                'rated_queries': 0,
+                'tool_usage': {}
+            }
+
+    def export_for_rlhf(self, output_path: Optional[Path] = None) -> Path:
+        """
+        Export logs in RLHF training format.
+
+        Format:
+        [
+            {
+                "prompt": "user query",
+                "completion": "ai response",
+                "rating": 5,
+                "feedback": "Great!",
+                "tools_used": ["list_grants"],
+                "timestamp": "2025-10-23T10:15:30"
+            },
+            ...
+        ]
+
+        Args:
+            output_path: Optional custom output path
+
+        Returns:
+            Path to exported file
+        """
+        if output_path is None:
+            today = datetime.now().strftime("%Y%m%d")
+            output_path = self.log_dir / f"rlhf_data_{today}.json"
+
+        rlhf_data = []
+
+        try:
+            # Read from JSONL
+            with open(self.jsonl_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    entry = json.loads(line)
+
+                    # Only include successful interactions
+                    if not entry.get('success', False):
+                        continue
+
+                    rlhf_entry = {
+                        'prompt': entry['user_query'],
+                        'completion': entry['ai_response'],
+                        'rating': entry.get('rating'),
+                        'feedback': entry.get('feedback'),
+                        'tools_used': entry.get('tools_called', []),
+                        'timestamp': entry['timestamp'],
+                        'response_time_ms': entry.get('response_time_ms'),
+                        'model': entry.get('model')
+                    }
+
+                    rlhf_data.append(rlhf_entry)
+
+            # Write RLHF format
+            with open(output_path, 'w', encoding='utf-8') as f:
+                json.dump(rlhf_data, f, indent=2, ensure_ascii=False)
+
+            logger.info(f"Exported {len(rlhf_data)} interactions to {output_path}")
+            return output_path
+
+        except Exception as e:
+            logger.error(f"Failed to export RLHF data: {e}")
+            raise
+
+
+# Global singleton instance
+_query_logger: Optional[QueryLogger] = None
+
+
+def get_query_logger(log_dir: str = "logs") -> QueryLogger:
+    """
+    Get or create the global query logger instance.
+
+    Args:
+        log_dir: Directory to store log files
+
+    Returns:
+        QueryLogger instance
+    """
+    global _query_logger
+    if _query_logger is None:
+        _query_logger = QueryLogger(log_dir=log_dir)
+    return _query_logger
+
+
+# Quick self-test
+if __name__ == "__main__":
+    logger = get_query_logger(log_dir="_out/test_logs")
+
+    # Log a test interaction
+    logger.log_interaction(
+        user_query="What grants are available for batteries?",
+        ai_response="Here are the battery-related grants: 1. Battery Innovation Grant...",
+        tools_called=["list_grants"],
+        response_time_ms=1500,
+        success=True,
+        rating=5,
+        feedback="Very helpful!",
+        model="gpt-5-mini"
+    )
+
+    # Get stats
+    stats = logger.get_stats()
+    print("Statistics:", json.dumps(stats, indent=2))
+
+    # Export for RLHF
+    output = logger.export_for_rlhf()
+    print(f"Exported to: {output}")

analyzer/utils/text.py

@@ -0,0 +1,57 @@
+"""
+utils/text.py — small text helpers (no hard deps)
+
+- clean(s): collapse whitespace
+- to_number(s): parse first numeric like "£1,234.50" -> 1234.5
+- extract_numbers(s): list of floats found in text
+- safe_truncate_chars(s, n): hard char limit with ellipsis
+- safe_truncate_tokens(s, max_tokens): uses tiktoken if installed; else char fallback
+"""
+from __future__ import annotations
+import re
+from typing import Any, List, Optional
+
+_NUM_RE = re.compile(r"-?\d+(?:\.\d+)?")
+
+def clean(s: Any) -> str:
+    return re.sub(r"\s+", " ", str(s or "")).strip()
+
+def to_number(x: Any) -> Optional[float]:
+    if x is None:
+        return None
+    if isinstance(x, (int, float)):
+        return float(x)
+    s = str(x).replace(",", "").replace("£", "").strip()
+    m = _NUM_RE.findall(s)
+    return float(m[0]) if m else None
+
+def extract_numbers(s: Any) -> List[float]:
+    return [float(m) for m in _NUM_RE.findall(str(s or ""))]
+
+def safe_truncate_chars(s: str, n: int) -> str:
+    s = s or ""
+    if len(s) <= n:
+        return s
+    return s[: max(0, n - 1)] + "…"
+
+def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-5-mini") -> str:
+    """
+    Best effort token truncation. If tiktoken is available, use it;
+    otherwise approximate by ~4 chars/token heuristic.
+    """
+    s = s or ""
+    try:
+        import tiktoken  # type: ignore
+        # Use o200k_base encoding for GPT-5 models (fallback to cl100k_base for GPT-4)
+        try:
+            enc = tiktoken.get_encoding("o200k_base")
+        except:
+            enc = tiktoken.get_encoding("cl100k_base")
+        toks = enc.encode(s)
+        if len(toks) <= max_tokens:
+            return s
+        toks = toks[:max_tokens]
+        return enc.decode(toks)
+    except Exception:
+        # rough fallback: ~4 chars per token
+        return safe_truncate_chars(s, max_tokens * 4)

analyzer/utils/validation.py

@@ -0,0 +1,204 @@
+"""Input validation with proper error raising."""
+import re
+from typing import Optional
+from .errors import ValidationError
+
+
+def validate_grant_id(gid: str) -> str:
+    """
+    Validate and normalize grant ID.
+
+    Args:
+        gid: Grant ID in any format ("2315", "competition-2315", etc.)
+
+    Returns:
+        Normalized ID (just the numeric part without prefix)
+
+    Raises:
+        ValidationError: If ID format is invalid
+
+    Example:
+        >>> validate_grant_id("2315")
+        '2315'
+        >>> validate_grant_id("competition-2315")
+        '2315'
+        >>> validate_grant_id("invalid")
+        ValidationError: Invalid grant ID format: 'invalid'
+    """
+    if not gid:
+        raise ValidationError("Grant ID cannot be empty")
+
+    gid = str(gid).strip()
+
+    # Extract numeric part
+    match = re.match(r'^(?:comp(?:etition)?-|grant-)?(\d{3,7})$', gid, re.I)
+    if not match:
+        raise ValidationError(
+            f"Invalid grant ID format: '{gid}'. "
+            f"Expected: '2315' or 'competition-2315'"
+        )
+
+    numeric_id = match.group(1)
+    return numeric_id
+
+
+def validate_url(url: str, *, allowed_hosts: Optional[set] = None) -> str:
+    """
+    Validate URL and check against allowlist.
+
+    Args:
+        url: URL to validate
+        allowed_hosts: Optional set of allowed hostnames
+
+    Returns:
+        Validated URL (unchanged)
+
+    Raises:
+        ValidationError: If URL is invalid or not allowed
+    """
+    from urllib.parse import urlparse
+
+    if not url:
+        raise ValidationError("URL cannot be empty")
+
+    url = str(url).strip()
+
+    if not url.startswith(('http://', 'https://')):
+        raise ValidationError(
+            f"URL must start with http:// or https://: {url}"
+        )
+
+    try:
+        parsed = urlparse(url)
+    except Exception as e:
+        raise ValidationError(f"Malformed URL: {url}") from e
+
+    if not parsed.netloc:
+        raise ValidationError(f"URL has no hostname: {url}")
+
+    if allowed_hosts and parsed.netloc not in allowed_hosts:
+        allowed_preview = ', '.join(list(allowed_hosts)[:3])
+        raise ValidationError(
+            f"URL host '{parsed.netloc}' not in allowlist. "
+            f"Allowed: {allowed_preview}..."
+        )
+
+    return url
+
+
+def sanitize_filename(name: str, max_length: int = 200) -> str:
+    """
+    Sanitize filename to prevent path traversal.
+
+    Args:
+        name: Original filename
+        max_length: Maximum length
+
+    Returns:
+        Safe filename
+
+    Example:
+        >>> sanitize_filename("../../../etc/passwd")
+        'etc_passwd'
+    """
+    if not name:
+        raise ValidationError("Filename cannot be empty")
+
+    # Remove path separators and dangerous chars
+    safe = re.sub(r'[^\w\-.]', '_', str(name))
+    safe = safe.strip('._')
+
+    if not safe:
+        raise ValidationError(f"Filename '{name}' produces empty result after sanitization")
+
+    return safe[:max_length]
+
+
+def validate_search_query(query: str, max_length: int = 500) -> str:
+    """
+    Validate search query.
+
+    Args:
+        query: Search query string
+        max_length: Maximum allowed length
+
+    Returns:
+        Validated query (stripped)
+
+    Raises:
+        ValidationError: If query is empty or too long
+    """
+    if not query:
+        raise ValidationError("Search query cannot be empty")
+
+    query = str(query).strip()
+
+    if not query:
+        raise ValidationError("Search query cannot be whitespace only")
+
+    if len(query) > max_length:
+        raise ValidationError(
+            f"Search query too long ({len(query)} chars, max {max_length})"
+        )
+
+    return query
+
+
+def validate_positive_int(value: any, name: str = "value") -> int:
+    """
+    Validate positive integer.
+
+    Args:
+        value: Value to validate
+        name: Name of parameter (for error messages)
+
+    Returns:
+        Validated integer
+
+    Raises:
+        ValidationError: If not a positive integer
+    """
+    try:
+        val = int(value)
+    except (TypeError, ValueError) as e:
+        raise ValidationError(f"{name} must be an integer, got {type(value).__name__}") from e
+
+    if val <= 0:
+        raise ValidationError(f"{name} must be positive, got {val}")
+
+    return val
+
+
+def validate_date_string(date_str: str, name: str = "date") -> str:
+    """
+    Validate ISO date string (YYYY-MM-DD).
+
+    Args:
+        date_str: Date string to validate
+        name: Name of parameter (for error messages)
+
+    Returns:
+        Validated date string
+
+    Raises:
+        ValidationError: If not valid ISO date format
+    """
+    if not date_str:
+        raise ValidationError(f"{name} cannot be empty")
+
+    date_str = str(date_str).strip()
+
+    # Check format
+    if not re.match(r'^\d{4}-\d{2}-\d{2}$', date_str):
+        raise ValidationError(
+            f"{name} must be in YYYY-MM-DD format, got: {date_str}"
+        )
+
+    # Validate actual date (catches invalid like 2025-13-45)
+    try:
+        from datetime import datetime
+        datetime.strptime(date_str, '%Y-%m-%d')
+    except ValueError as e:
+        raise ValidationError(f"Invalid date: {date_str}") from e
+
+    return date_str

requirements.txt

@@ -1,54 +1,22 @@
-# Core dependencies
-pandas>=2.0
-numpy>=1.26,<2.0
-openpyxl>=3.1
-xlsxwriter>=3.1
-
-# Machine learning / retrieval
-scikit-learn>=1.4
-rapidfuzz>=3.9
-
-# LLM providers
-openai>=1.40
-anthropic>=0.71
-
-# Token utilities
-tiktoken>=0.7
-
-# Web scraping and async
-beautifulsoup4>=4.12
-lxml>=4.9
-httpx>=0.27
-requests>=2.31
-aiohttp>=3.9
-aiofiles>=23.2
-
-# API and web framework
-fastapi>=0.100
-uvicorn>=0.24
-pydantic>=2.0,<2.12
-pydantic-settings>=2.4
-gradio>=5.0
-
-# Database
-pymongo>=4.6
-motor>=3.3
-
-# PDF processing (optional - comment out if not needed)
-PyMuPDF>=1.23
-pdfplumber>=0.10
-
-# Testing
-pytest>=7.4
-pytest-cov>=4.1
-pytest-asyncio>=0.21
-
-# Utilities
-python-dateutil>=2.8
-python-dotenv>=1.0
-python-json-logger>=3.0
-typer>=0.9
-
-# Web automation (optional - only if using crawler)
-# playwright>=1.40
-# Scrapy>=2.11
+# Hugging Face Spaces requirements
+# Optimized for cloud deployment
+# Note: Gradio is provided by HF Spaces when sdk: gradio is specified in README
+
+openai>=1.0.0
+anthropic>=0.71.0
+pandas>=2.0.0
+numpy>=1.24.0
+scikit-learn>=1.3.0
+requests>=2.31.0
+aiohttp>=3.9.0
+httpx>=0.27.0
+aiofiles>=23.2.0
+python-dotenv>=1.0.0
+beautifulsoup4>=4.12.0
+lxml>=5.0.0
+pydantic>=2.0,<3.0
+pydantic-settings>=2.0.0
+python-json-logger>=3.0.0
+
+# Note: MongoDB dependencies (pymongo, motor) removed for HF Spaces
+# as they're not needed for the core functionality