fix: Complete restoration of analyzer/ directory from c72a240

Restored the ENTIRE analyzer/ directory from commit c72a240, which was
the last known fully working Hugging Face Spaces deployment.

This includes:
- All chat tools and UI (demo_app.py with 7 preset questions)
- LLM client, config, data loaders
- Crawler/scraper tools
- All utilities and supporting modules

Every file is now exactly as it was in the working c72a240 deployment,
removing all changes that happened during the problematic merge/push cycle.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

analyzer/chat/company_analyzer.py

@@ -1,384 +0,0 @@
-"""
-Enhanced company analysis for grant matching v2.
-
-This module provides intelligent company profiling and grant matching based on:
-- Technology stack and industry sector
-- Company size and stage
-- Location and eligibility requirements
-- Funding amounts and project types
-"""
-from __future__ import annotations
-
-import logging
-import re
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Dict, List, Optional, Set, Any
-
-from ..net.fetcher import fetch_link
-from ..utils.text import clean
-from ..utils.dates import parse_date
-
-logger = logging.getLogger(__name__)
-
-
-# Technology keywords for sector detection
-TECH_KEYWORDS = {
-    "ai_ml": ["ai", "artificial intelligence", "machine learning", "ml", "deep learning",
-              "neural network", "llm", "gpt", "nlp", "computer vision", "agentic"],
-    "battery_ev": ["battery", "batteries", "electric vehicle", "ev", "electrification",
-                   "energy storage", "lithium", "zero emission"],
-    "biotech": ["biotech", "pharmaceutical", "drug discovery", "clinical", "medical device",
-                "diagnostic", "therapeutic", "genomic", "bioinformatics"],
-    "manufacturing": ["manufacturing", "production", "factory", "industrial", "assembly",
-                      "automation", "robotics", "supply chain"],
-    "software": ["software", "saas", "platform", "application", "app", "digital", "cloud"],
-    "green_tech": ["sustainability", "renewable", "green energy", "climate", "carbon",
-                   "environmental", "circular economy", "net zero"],
-    "aerospace": ["aerospace", "aviation", "aircraft", "satellite", "space", "drone"],
-    "quantum": ["quantum computing", "quantum", "qubit"],
-}
-
-# Company size indicators
-SIZE_INDICATORS = {
-    "startup": ["startup", "founded in 202", "seed", "pre-seed", "early stage"],
-    "scale_up": ["scale-up", "scaleup", "series a", "series b", "growing", "expansion"],
-    "sme": ["small business", "sme", "small to medium", "limited", "ltd"],
-    "enterprise": ["enterprise", "corporation", "plc", "publicly traded", "fortune"],
-}
-
-# Location keywords for UK eligibility
-UK_LOCATIONS = [
-    "uk", "united kingdom", "london", "manchester", "birmingham", "glasgow", "edinburgh",
-    "bristol", "leeds", "liverpool", "cardiff", "belfast", "scotland", "wales", "england",
-    "northern ireland", "britain", "british"
-]
-
-
-@dataclass
-class CompanyProfile:
-    """Extracted company profile for grant matching."""
-    url: str
-    text_content: str
-
-    # Detected attributes
-    sectors: Set[str]
-    tech_stack: Set[str]
-    company_size: Optional[str]
-    is_uk_based: bool
-    keywords: Set[str]
-
-    # Inferred characteristics
-    appears_r_and_d_focused: bool
-    mentions_funding: bool
-
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for JSON serialization."""
-        return {
-            "url": self.url,
-            "sectors": list(self.sectors),
-            "tech_stack": list(self.tech_stack),
-            "company_size": self.company_size,
-            "is_uk_based": self.is_uk_based,
-            "appears_r_and_d_focused": self.appears_r_and_d_focused,
-            "mentions_funding": self.mentions_funding,
-            "keywords": list(self.keywords)[:20],  # Limit for readability
-        }
-
-
-def extract_company_profile(company_url: str) -> Optional[CompanyProfile]:
-    """
-    Extract detailed company profile from website.
-
-    Args:
-        company_url: URL of company website
-
-    Returns:
-        CompanyProfile with extracted attributes, or None if fetch fails
-    """
-    logger.info(f"Extracting company profile from: {company_url}")
-
-    # Use the existing fetcher
-    result = fetch_link(company_url)
-
-    if not result.get("ok"):
-        logger.error(f"Failed to fetch {company_url}: {result.get('error')}")
-        return None
-
-    text = result.get("text", "")
-    text_lower = text.lower()
-
-    # Detect technology sectors
-    sectors = set()
-    tech_stack = set()
-    for sector, keywords in TECH_KEYWORDS.items():
-        for keyword in keywords:
-            if keyword in text_lower:
-                sectors.add(sector)
-                tech_stack.add(keyword)
-
-    # Detect company size
-    company_size = None
-    for size_type, indicators in SIZE_INDICATORS.items():
-        for indicator in indicators:
-            if indicator in text_lower:
-                company_size = size_type
-                break
-        if company_size:
-            break
-
-    # Check UK location
-    is_uk_based = any(loc in text_lower for loc in UK_LOCATIONS)
-
-    # Extract meaningful keywords (simple approach)
-    words = re.findall(r'\b[a-z]{4,}\b', text_lower)
-    # Filter out common words
-    common_words = {"about", "their", "with", "from", "that", "this", "have", "more",
-                    "what", "when", "where", "which", "they", "would", "could", "should"}
-    keywords = set(w for w in words if w not in common_words)
-
-    # Detect R&D focus
-    r_and_d_indicators = ["research", "development", "innovation", "r&d", "patent",
-                          "prototype", "pilot", "feasibility", "experimental"]
-    appears_r_and_d_focused = any(ind in text_lower for ind in r_and_d_indicators)
-
-    # Check if they mention funding
-    funding_indicators = ["funding", "investment", "grant", "raise", "capital", "finance"]
-    mentions_funding = any(ind in text_lower for ind in funding_indicators)
-
-    profile = CompanyProfile(
-        url=company_url,
-        text_content=text[:5000],  # Keep first 5000 chars for analysis
-        sectors=sectors,
-        tech_stack=tech_stack,
-        company_size=company_size,
-        is_uk_based=is_uk_based,
-        keywords=keywords,
-        appears_r_and_d_focused=appears_r_and_d_focused,
-        mentions_funding=mentions_funding,
-    )
-
-    logger.info(f"Profile extracted: {len(sectors)} sectors, size={company_size}, UK={is_uk_based}")
-    return profile
-
-
-@dataclass
-class GrantMatch:
-    """Represents a grant match with scoring and reasoning."""
-    grant_id: str
-    grant_title: str
-    match_score: float  # 0-100
-    match_category: str  # "perfect", "strong", "potential"
-    reasons: List[str]
-    concerns: List[str]
-    deadline: Optional[str]
-    funding_max: Optional[float]
-
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary."""
-        return {
-            "grant_id": self.grant_id,
-            "grant_title": self.grant_title,
-            "match_score": round(self.match_score, 1),
-            "match_category": self.match_category,
-            "reasons": self.reasons,
-            "concerns": self.concerns if self.concerns else None,
-            "deadline": self.deadline,
-            "funding_max": self.funding_max,
-        }
-
-
-def calculate_grant_status(grant: Dict[str, Any]) -> str:
-    """Calculate grant status based on dates."""
-    today = datetime.now()
-    close_date = parse_date(grant.get("close_date") or grant.get("deadline"))
-    open_date = parse_date(grant.get("open_date"))
-
-    if not close_date:
-        return "unknown"
-    if close_date < today:
-        return "closed"
-    if open_date and open_date > today:
-        return "upcoming"
-    return "open"
-
-
-def score_grant_match(
-    profile: CompanyProfile,
-    grant: Dict[str, Any]
-) -> GrantMatch:
-    """
-    Score how well a grant matches a company profile.
-
-    Returns:
-        GrantMatch with score, category, reasons, and concerns
-    """
-    grant_id = grant.get("id") or grant.get("competition_id") or "unknown"
-    grant_title = grant.get("title", "(untitled)")
-
-    # Prepare grant text for analysis
-    grant_text = " ".join([
-        str(grant.get("title", "")),
-        str(grant.get("summary", "")),
-        str(grant.get("scope", "")),
-        str(grant.get("eligibility", "")),
-    ]).lower()
-
-    score = 0.0
-    reasons = []
-    concerns = []
-
-    # 1. Sector/Technology alignment (40 points max)
-    sector_matches = []
-    for sector in profile.sectors:
-        sector_keywords = TECH_KEYWORDS.get(sector, [])
-        for keyword in sector_keywords:
-            if keyword in grant_text:
-                sector_matches.append(keyword)
-
-    if sector_matches:
-        sector_score = min(40, len(sector_matches) * 10)
-        score += sector_score
-        reasons.append(f"Strong sector alignment: {', '.join(set(sector_matches[:3]))}")
-
-    # 2. UK eligibility (20 points if UK-based)
-    grant_status = calculate_grant_status(grant)
-
-    if profile.is_uk_based:
-        score += 20
-        reasons.append("UK-based company (eligible for Innovate UK)")
-    else:
-        concerns.append("Company may not be UK-based (verify eligibility)")
-
-    # 3. Grant status (20 points if open)
-    if grant_status == "open":
-        score += 20
-        reasons.append(f"Grant is currently open")
-    elif grant_status == "upcoming":
-        score += 15
-        reasons.append(f"Grant opens soon")
-    elif grant_status == "closed":
-        score -= 30
-        concerns.append("Grant deadline has passed")
-
-    # 4. Company size/stage fit (10 points)
-    eligibility_text = str(grant.get("eligibility", "")).lower()
-    if profile.company_size:
-        size_mentioned = profile.company_size in eligibility_text
-        if "sme" in eligibility_text or "small" in eligibility_text:
-            if profile.company_size in ["startup", "sme", "scale_up"]:
-                score += 10
-                reasons.append(f"Good fit for {profile.company_size}s")
-        elif size_mentioned:
-            score += 10
-            reasons.append(f"Mentions {profile.company_size}s")
-
-    # 5. R&D focus alignment (10 points)
-    if profile.appears_r_and_d_focused:
-        r_and_d_in_grant = any(word in grant_text for word in ["research", "development", "innovation", "r&d"])
-        if r_and_d_in_grant:
-            score += 10
-            reasons.append("R&D-focused opportunity (matches company profile)")
-
-    # 6. Funding amount considerations
-    funding_max = grant.get("funding_max") or grant.get("max_award")
-    if funding_max:
-        try:
-            funding_val = float(str(funding_max).replace(",", "").replace("£", ""))
-            if funding_val > 1000000:  # £1M+
-                if profile.company_size == "startup":
-                    concerns.append(f"Large grant (£{funding_val:,.0f}) - may require significant match funding")
-        except:
-            pass
-
-    # Determine match category
-    if score >= 70:
-        category = "perfect"
-    elif score >= 50:
-        category = "strong"
-    elif score >= 30:
-        category = "potential"
-    else:
-        category = "weak"
-
-    return GrantMatch(
-        grant_id=grant_id,
-        grant_title=grant_title,
-        match_score=score,
-        match_category=category,
-        reasons=reasons,
-        concerns=concerns if concerns else [],
-        deadline=grant.get("deadline") or grant.get("close_date"),
-        funding_max=funding_max,
-    )
-
-
-def analyze_company_for_grants_v2(
-    company_url: str,
-    grants: List[Dict[str, Any]],
-    limit: int = 10
-) -> Dict[str, Any]:
-    """
-    Enhanced company analysis with smart grant matching.
-
-    Args:
-        company_url: URL of company website
-        grants: List of available grants to match against
-        limit: Maximum number of recommendations per category
-
-    Returns:
-        Dict with company profile, perfect matches, strong matches, and exclusion reasons
-    """
-    logger.info(f"Starting enhanced company analysis for: {company_url}")
-
-    # Extract company profile
-    profile = extract_company_profile(company_url)
-    if not profile:
-        return {
-            "error": "Failed to extract company profile from URL",
-            "company_url": company_url,
-        }
-
-    # Score all grants
-    all_matches = []
-    for grant in grants:
-        match = score_grant_match(profile, grant)
-        all_matches.append(match)
-
-    # Sort by score
-    all_matches.sort(key=lambda m: m.match_score, reverse=True)
-
-    # Categorize matches
-    perfect_matches = [m for m in all_matches if m.match_category == "perfect"][:limit]
-    strong_matches = [m for m in all_matches if m.match_category == "strong"][:limit]
-    potential_matches = [m for m in all_matches if m.match_category == "potential"][:limit]
-
-    # Explain why others were excluded (top reasons)
-    excluded = [m for m in all_matches if m.match_category == "weak"]
-    exclusion_reasons = {}
-    for match in excluded[:10]:  # Analyze top 10 excluded
-        for concern in match.concerns:
-            if concern not in exclusion_reasons:
-                exclusion_reasons[concern] = 0
-            exclusion_reasons[concern] += 1
-
-    # Sort exclusion reasons by frequency
-    top_exclusions = sorted(exclusion_reasons.items(), key=lambda x: x[1], reverse=True)[:5]
-
-    return {
-        "company_url": company_url,
-        "company_profile": profile.to_dict(),
-        "perfect_matches": [m.to_dict() for m in perfect_matches],
-        "strong_matches": [m.to_dict() for m in strong_matches],
-        "worth_considering": [m.to_dict() for m in potential_matches],
-        "why_not_others": {
-            "top_exclusion_reasons": [reason for reason, count in top_exclusions],
-            "total_excluded": len(excluded),
-        },
-        "summary": {
-            "total_analyzed": len(all_matches),
-            "perfect_matches_count": len(perfect_matches),
-            "strong_matches_count": len(strong_matches),
-            "potential_matches_count": len(potential_matches),
-        }
-    }

analyzer/chat/query_router.py

@@ -13,15 +13,9 @@ This helps reduce redundant information requests and improves UX.
 from __future__ import annotations
 from dataclasses import dataclass
 from typing import Dict, Optional, Tuple, List
-import json, re, hashlib, time, logging
+import json, re
 
-logger = logging.getLogger(__name__)
-
-_INTENTS = {"list", "summarize", "compare", "deadlines", "search", "general", "get_grant", "list_grants"}
-
-# Routing cache with 24-hour TTL
-_routing_cache: Dict[str, Tuple[Dict, float]] = {}
-_CACHE_TTL = 86400  # 24 hours in seconds
+_INTENTS = {"list", "summarize", "compare", "deadlines", "search", "general"}
 
 # Accept "competition-2315", "2315", "comp-2315"
 _ID_RE = re.compile(r"(?:comp(?:etition)?-)?([0-9]{3,7})", re.IGNORECASE)
@@ -112,93 +106,14 @@ def _detect_status_filter(text: str) -> Optional[str]:
     return None
 
 
-def classify_complexity(query: str) -> str:
-    """
-    Classify query complexity to select appropriate model.
-
-    Returns:
-        'simple': Use gpt-5-mini for simple translation/explanation tasks
-        'complex': Use gpt-5 for detailed analysis/comparisons
-        'medium': Default, use gpt-5-mini
-    """
-    low = query.lower()
-
-    # Simple queries: translation, explanation, layman's terms
-    simple_keywords = {"translate", "explain", "simple", "layman", "what is", "define"}
-    if any(kw in low for kw in simple_keywords):
-        return 'simple'
-
-    # Complex queries: detailed analysis, comparisons
-    complex_keywords = {"compare", "analyze", "detailed", "comprehensive", "in-depth"}
-    if any(kw in low for kw in complex_keywords):
-        return 'complex'
-
-    # Default to medium complexity
-    return 'medium'
-
-
-def _route_with_llm(text: str) -> Optional[str]:
+def route(text: str, *, use_llm: bool = False) -> Dict:
     """
-    Route query using GPT-5-nano for intelligent intent classification.
-
-    Returns:
-        Intent string or None if LLM fails
-    """
-    try:
-        from ..llm_client import LLMClient
-        from ..config import load_config
-
-        # Initialize LLM client with router model
-        config = load_config()
-        client = LLMClient(config)
-
-        if not client.is_ready():
-            return None
-
-        # Build routing prompt
-        prompt = f"""Classify this query into ONE of these intents: search, summarize, compare, get_grant, list_grants, deadlines, general.
-
-Query: {text}
-
-Respond with ONLY the intent word, nothing else."""
-
-        # Get LLM classification with routing parameters
-        messages = [
-            {"role": "system", "content": "You are a query intent classifier. Respond with only one word."},
-            {"role": "user", "content": prompt}
-        ]
-
-        response = client.chat(
-            messages,
-            model_type="router",  # Use gpt-5-nano
-            verbosity="low",
-            reasoning_effort="minimal",
-            max_tokens=10,
-            temperature=0.1
-        )
-
-        # Parse single word response
-        intent = response.strip().lower()
-
-        # Validate intent
-        if intent in _INTENTS:
-            logger.info(f"🤖 LLM routing: '{text[:50]}...' -> {intent}")
-            return intent
-        else:
-            logger.warning(f"LLM returned invalid intent: {intent}")
-            return None
-
-    except Exception as e:
-        logger.warning(f"LLM routing failed: {e}")
-        return None
-
-
-def _route_with_regex(text: str) -> Dict:
-    """
-    Fallback regex-based routing (original implementation).
+    Route a user query to the appropriate intent handler.
 
-    Returns:
-        Routed dict with intent, args, and confidence
+    Improved to handle:
+    - Synonyms (grants = calls = opportunities = funding)
+    - Status filters (open, closed, upcoming)
+    - Variations of the same intent
     """
     t = text.strip()
     low = t.lower()
@@ -212,9 +127,10 @@ def _route_with_regex(text: str) -> Dict:
         kw = t.split(" ", 1)[1].strip() if " " in t else ""
 
         # Remove punctuation
+        import re
         kw = re.sub(r'[?!.,;:]', '', kw).strip()
 
-        # Clean up filler words
+        # Clean up filler words like "me", "please", "all", "available", "is", "are"
         filler = {
             "me", "please", "show", "list", "all", "available", "open", "closed", "upcoming",
             "grants", "grant", "opportunities", "opportunity", "funding", "calls", "call",
@@ -224,112 +140,51 @@ def _route_with_regex(text: str) -> Dict:
         kw = " ".join(kw_tokens) if kw_tokens else ""
 
         args = {}
+        # Only add keyword if we have a real keyword (not just grant-related filler)
         if kw:
             args["keyword"] = kw
+
         if status_filter:
             args["status"] = status_filter
 
+        # Return ALL grants if just listing (no limit = all)
         args["limit"] = None
-        return Routed("list", args, 0.75).to_dict()
+        return Routed("list", args, 0.95).to_dict()
 
     if low.startswith("summarize") or low.startswith("summarise"):
         ids, _ = _extract_ids(t)
         if len(ids) >= 2:
-            return Routed("compare", {"grant_id_a": f"competition-{ids[0]}", "grant_id_b": f"competition-{ids[1]}"}, 0.85).to_dict()
+            return Routed("compare", {"grant_id_a": f"competition-{ids[0]}", "grant_id_b": f"competition-{ids[1]}"}, 0.95).to_dict()
         if len(ids) == 1:
-            return Routed("summarize", {"grant_id": f"competition-{ids[0]}"}, 0.85).to_dict()
+            return Routed("summarize", {"grant_id": f"competition-{ids[0]}"}, 0.95).to_dict()
+        # no IDs → treat remainder as search
         kw = t.split(" ", 1)[1].strip() if " " in t else ""
-        return Routed("search", {"keyword": kw, "limit": None}, 0.6).to_dict()
+        return Routed("search", {"keyword": kw, "limit": None}, 0.6).to_dict()  # FIXED: No limit = return all
 
     # Deadline queries
     if any(p in low for p in ["deadline", "close date", "when is it due", "when do i need to apply", "application deadline"]):
-        return Routed("deadlines", {"n": None}, 0.75).to_dict()
+        return Routed("deadlines", {"n": None}, 0.85).to_dict()  # Return ALL deadlines
 
     # compare / vs / versus / between → compare two grants
     if "compare" in low or " vs " in low or "versus" in low or "between" in low:
         ids, residual = _extract_ids(t)
+        facet = residual.strip()
         if len(ids) >= 2:
             return Routed(
                 "compare",
                 {"grant_id_a": f"competition-{ids[0]}", "grant_id_b": f"competition-{ids[1]}"},
-                0.82
+                0.92
             ).to_dict()
         if len(ids) == 1:
             return Routed("summarize", {"grant_id": f"competition-{ids[0]}"}, 0.7).to_dict()
 
-    # Natural search
+    # Natural search (lower confidence, but still strong)
     if any(w in low for w in ["grant", "funding", "competition", "call", "apply", "what", "which", "find", "search"]):
         kw = _keywords_from_question(t)
-        return Routed("search", {"keyword": kw, "limit": None}, 0.65).to_dict()
+        return Routed("search", {"keyword": kw, "limit": None}, 0.75).to_dict()  # FIXED: No limit = return all
 
     return Routed("general", {"question": t}, 0.5).to_dict()
 
-
-def route(text: str, *, use_llm: bool = True) -> Dict:
-    """
-    Route a user query to the appropriate intent handler.
-
-    Uses GPT-5-nano for intelligent routing with regex fallback.
-    Results are cached for 24 hours for performance.
-
-    Args:
-        text: User query text
-        use_llm: If True, use GPT-5-nano for routing (default: True)
-
-    Returns:
-        Dict with intent, args, and confidence
-    """
-    t = text.strip()
-
-    # Check cache first
-    cache_key = hashlib.md5(t.lower().encode()).hexdigest()
-    if cache_key in _routing_cache:
-        cached_result, cached_time = _routing_cache[cache_key]
-        if time.time() - cached_time < _CACHE_TTL:
-            logger.debug(f"📦 Cache HIT for routing: '{t[:50]}...'")
-            return cached_result
-
-    # Try LLM routing first (if enabled)
-    result = None
-    if use_llm:
-        llm_intent = _route_with_llm(t)
-        if llm_intent:
-            # Extract args based on intent
-            args = {}
-            ids, _ = _extract_ids(t)
-
-            if llm_intent == "summarize" and len(ids) == 1:
-                args["grant_id"] = f"competition-{ids[0]}"
-            elif llm_intent == "compare" and len(ids) >= 2:
-                args["grant_id_a"] = f"competition-{ids[0]}"
-                args["grant_id_b"] = f"competition-{ids[1]}"
-            elif llm_intent in ("search", "list_grants"):
-                kw = _keywords_from_question(t)
-                if kw:
-                    args["keyword"] = kw
-                status = _detect_status_filter(t)
-                if status:
-                    args["status"] = status
-                args["limit"] = None
-            elif llm_intent == "get_grant" and len(ids) == 1:
-                args["grant_id"] = f"competition-{ids[0]}"
-            elif llm_intent == "deadlines":
-                args["n"] = None
-            else:
-                args["question"] = t
-
-            result = Routed(llm_intent, args, 0.95).to_dict()
-
-    # Fall back to regex if LLM failed or disabled
-    if result is None:
-        logger.debug(f"🔧 Using regex fallback for: '{t[:50]}...'")
-        result = _route_with_regex(t)
-
-    # Cache the result
-    _routing_cache[cache_key] = (result, time.time())
-
-    return result
-
 # Self-test
 if __name__ == "__main__":
     tests = [

analyzer/chat/run_chat_llm.py

@@ -108,6 +108,16 @@ def _dispatch_tool_call(tools: ChatTools, tool_name: str, tool_args: Dict[str, A
                 "query": tool_args.get("query")
             }
 
+        elif tool_name == "search_past_winners":
+            return {
+                "results": tools.search_past_winners(
+                    keyword=tool_args.get("keyword"),
+                    competition=tool_args.get("competition"),
+                    limit=tool_args.get("limit")
+                ),
+                "query": tool_args.get("keyword") or tool_args.get("competition") or "all"
+            }
+
         else:
             return {"error": f"Unknown tool: {tool_name}"}
 

analyzer/config.py

@@ -32,7 +32,7 @@ from typing import Literal, Optional
 Provider = Literal["openai", "anthropic"]
 
 DEFAULT_MODELS = {
-    "openai": "gpt-5-mini",
+    "openai": "gpt-4.1-mini",   # safe default; override to gpt-5-mini if enabled for your key
     "anthropic": "claude-3-5-haiku-20241022",  # fast, low-cost Claude
 }
 
@@ -45,11 +45,6 @@ class Config:
     openai_api_key: Optional[str] = None
     anthropic_api_key: Optional[str] = None
 
-    # Model-specific configurations for different use cases
-    model_router: str = "gpt-5-nano"
-    model_translator: str = "gpt-5-mini"
-    model_analyzer: str = "gpt-5"
-
     temperature: float = 0.2
     max_output_tokens: int = 800
     timeout_s: float = 30.0
@@ -91,9 +86,6 @@ def load_config() -> Config:
         model=model,
         openai_api_key=_env("OPENAI_API_KEY"),
         anthropic_api_key=_env("ANTHROPIC_API_KEY"),
-        model_router=_env("LLM_MODEL_ROUTER", "gpt-5-nano"),
-        model_translator=_env("LLM_MODEL_TRANSLATOR", "gpt-5-mini"),
-        model_analyzer=_env("LLM_MODEL_ANALYZER", "gpt-5"),
         temperature=float(_env("LLM_TEMPERATURE", "0.2")),
         max_output_tokens=int(_env("LLM_MAX_OUTPUT_TOKENS", "800")),
         timeout_s=float(_env("LLM_TIMEOUT_S", "30")),

analyzer/crawler/discover_grants.py

@@ -1,306 +0,0 @@
-"""
-Discover and fetch new grants from Innovate UK website.
-
-This module:
-1. Fetches the Innovate UK competition search/listing page
-2. Extracts all available grant URLs
-3. Fetches each grant's details using snapshot.py
-4. Saves to snapshots directory
-"""
-
-import logging
-import asyncio
-import json
-import re
-from pathlib import Path
-from typing import List, Tuple, Optional, Set
-from urllib.parse import urljoin, urlparse
-from datetime import datetime, UTC
-from playwright.async_api import async_playwright
-from bs4 import BeautifulSoup
-
-logger = logging.getLogger(__name__)
-
-# Innovate UK service base URL
-IUK_BASE = "https://apply-for-innovation-funding.service.gov.uk"
-COMPETITIONS_URL = f"{IUK_BASE}/competition/search"
-
-
-async def discover_grant_urls(max_retries: int = 2) -> List[str]:
-    """
-    Discover all available grant overview URLs from Innovate UK.
-
-    Returns:
-        List of grant overview URLs
-    """
-    logger.info(f"Discovering grants from {COMPETITIONS_URL}")
-
-    async with async_playwright() as pw:
-        browser = await pw.chromium.launch(
-            headless=True,
-            args=["--disable-dev-shm-usage"]
-        )
-        context = await browser.new_context(
-            user_agent=(
-                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-                "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
-            ),
-            locale="en-GB",
-            timezone_id="Europe/London",
-        )
-        page = await context.new_page()
-        page.set_default_timeout(30000)
-
-        for attempt in range(max_retries):
-            try:
-                await page.goto(COMPETITIONS_URL, wait_until="domcontentloaded")
-                await page.wait_for_load_state("networkidle")
-                break
-            except Exception as e:
-                if attempt == max_retries - 1:
-                    logger.error(f"Failed to fetch competitions page: {e}")
-                    await context.close()
-                    await browser.close()
-                    return []
-                logger.warning(f"Attempt {attempt + 1} failed, retrying...")
-
-        html = await page.content()
-        await context.close()
-        await browser.close()
-
-    # Parse URLs from HTML
-    urls = _extract_grant_urls(html)
-    logger.info(f"Discovered {len(urls)} grants")
-
-    return urls
-
-
-def _extract_grant_urls(html: str) -> List[str]:
-    """
-    Extract all grant overview URLs from the competitions listing page.
-
-    Looks for links matching pattern: /competition/{id}/overview/{uuid}
-    """
-    soup = BeautifulSoup(html, "lxml")
-    urls = []
-
-    # Find all links that match the overview pattern
-    pattern = re.compile(r'/competition/(\d+)/overview/([0-9a-f\-]{8,})', re.I)
-
-    for link in soup.find_all("a", href=True):
-        href = link.get("href", "")
-        if pattern.search(href):
-            # Make absolute URL
-            full_url = urljoin(IUK_BASE, href)
-            if full_url not in urls:
-                urls.append(full_url)
-
-    # Also check for links in data attributes or javascript
-    for elem in soup.find_all(["a", "div", "li"], {"data-href": True}):
-        href = elem.get("data-href", "")
-        if pattern.search(href):
-            full_url = urljoin(IUK_BASE, href)
-            if full_url not in urls:
-                urls.append(full_url)
-
-    return urls
-
-
-async def fetch_grant_snapshot(url: str, output_dir: Path) -> Optional[str]:
-    """
-    Fetch a single grant's snapshot and save to JSON.
-
-    Args:
-        url: Grant overview URL
-        output_dir: Directory to save snapshot JSON
-
-    Returns:
-        Filename of saved snapshot, or None if failed
-    """
-    try:
-        from .snapshot import fetch_sections_from_overview, parse_deeplink
-
-        # Extract grant ID from URL
-        match = re.search(r'/competition/(\d+)/', url)
-        if not match:
-            logger.warning(f"Could not extract grant ID from {url}")
-            return None
-
-        grant_id = match.group(1)
-        output_path = output_dir / f"competition-{grant_id}.json"
-
-        # Skip if already exists
-        if output_path.exists():
-            logger.debug(f"Grant {grant_id} already exists, skipping")
-            return None
-
-        logger.info(f"Fetching grant {grant_id}...")
-        html, sections = await fetch_sections_from_overview(url)
-
-        # Parse dates and funding from the fetched content
-        from .snapshot import (
-            parse_dates_singleline,
-            extract_funding,
-            _find_duration_months,
-            pick_open_close_from_milestones,
-            derive_aux_dates,
-            clean_title
-        )
-
-        # Parse dates
-        dates_text = sections.get("dates_raw", "") or ""
-        milestones = parse_dates_singleline(dates_text)
-
-        # Derive open/close from milestones
-        open_date, close_date = pick_open_close_from_milestones(milestones)
-
-        # If either missing, try scanning all text
-        if not open_date or not close_date:
-            all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
-            extra = parse_dates_singleline(all_text)
-            seen = {(m["label_raw"], m["date_iso"], m.get("date_iso_end")) for m in milestones}
-            for m2 in extra:
-                key = (m2["label_raw"], m2["date_iso"], m2.get("date_iso_end"))
-                if key not in seen:
-                    milestones.append(m2)
-                    seen.add(key)
-            od2, cd2 = pick_open_close_from_milestones(milestones)
-            open_date = open_date or od2
-            close_date = close_date or cd2
-
-        notify_date, project_start_from = derive_aux_dates(milestones)
-
-        # Parse funding
-        funding = extract_funding(sections)
-
-        # Parse duration
-        dur_min, dur_max = _find_duration_months(sections.get("eligibility_raw", "") or "")
-        if dur_min is None or dur_max is None:
-            all_text_for_duration = " ".join(v for v in sections.values() if isinstance(v, str) and v)
-            dur_min, dur_max = _find_duration_months(all_text_for_duration)
-        duration_months = {"min": dur_min, "max": dur_max}
-
-        # Extract title
-        raw_title = sections.get("summary_raw", "").split("\n")[0] if sections.get("summary_raw") else ""
-        title = clean_title(raw_title)
-
-        # Create snapshot
-        snapshot = {
-            "id": f"competition-{grant_id}",
-            "competition_id": grant_id,
-            "url": url,
-            "title": title,
-            "programme": "",
-            "round": "",
-            "open_date": open_date,
-            "close_date": close_date,
-            "notify_date": notify_date,
-            "project_start_from": project_start_from,
-            "funding": funding,
-            "duration_months": duration_months,
-            "sections": sections,
-            "pdfs": [],
-            "summaries": {},
-            "extracted": {"milestones": milestones},
-            "wonky": {"score": 0.0, "reasons": []},
-            "prev_round_refs": [],
-            "diff_summary": "",
-            "history_stats": {},
-            "created_at": datetime.now(UTC).isoformat(),
-            "updated_at": datetime.now(UTC).isoformat(),
-        }
-
-        # Save snapshot
-        output_path.write_text(
-            json.dumps(snapshot, indent=2),
-            encoding="utf-8"
-        )
-        logger.info(f"Saved grant {grant_id} snapshot")
-
-        return output_path.name
-
-    except Exception as e:
-        logger.error(f"Failed to fetch grant from {url}: {e}")
-        return None
-
-
-async def discover_and_fetch_grants(
-    output_dir: Path,
-    skip_existing: bool = True
-) -> Tuple[int, int, List[str]]:
-    """
-    Discover all grants and fetch new ones.
-
-    Args:
-        output_dir: Directory to save snapshots
-        skip_existing: Skip grants that already exist
-
-    Returns:
-        Tuple of (total_discovered, newly_fetched, new_filenames)
-    """
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Step 1: Discover all grant URLs
-    urls = await discover_grant_urls()
-    if not urls:
-        logger.warning("No grants discovered")
-        return 0, 0, []
-
-    # Step 2: Get existing grant IDs if skipping
-    existing_ids: Set[str] = set()
-    if skip_existing:
-        for json_file in output_dir.glob("competition-*.json"):
-            match = re.search(r'competition-(\d+)', json_file.name)
-            if match:
-                existing_ids.add(match.group(1))
-
-    # Step 3: Fetch new grants concurrently
-    new_files = []
-    tasks = []
-
-    for url in urls:
-        match = re.search(r'/competition/(\d+)/', url)
-        if match and match.group(1) in existing_ids:
-            logger.debug(f"Grant {match.group(1)} already exists")
-            continue
-
-        tasks.append(fetch_grant_snapshot(url, output_dir))
-
-    if tasks:
-        logger.info(f"Fetching {len(tasks)} new grants concurrently...")
-        results = await asyncio.gather(*tasks, return_exceptions=False)
-        new_files = [f for f in results if f is not None]
-
-    logger.info(
-        f"Discovery complete: {len(urls)} total, "
-        f"{len(new_files)} newly fetched"
-    )
-
-    return len(urls), len(new_files), new_files
-
-
-def main_sync(
-    output_dir: str = "data/snapshots",
-    skip_existing: bool = True
-) -> Tuple[int, int, List[str]]:
-    """
-    Synchronous wrapper for discovering and fetching grants.
-    """
-    output_path = Path(output_dir)
-    return asyncio.run(
-        discover_and_fetch_grants(output_path, skip_existing)
-    )
-
-
-if __name__ == "__main__":
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-
-    total, new, files = main_sync()
-    print(f"\n✓ Discovery complete:")
-    print(f"  Total discovered: {total}")
-    print(f"  Newly fetched: {new}")
-    if files:
-        print(f"  Files: {', '.join(files)}")

analyzer/crawler/scheduler.py

@@ -198,147 +198,6 @@ def rebuild_search_index(
         return False
 
 
-def generate_summaries_for_grants(
-    snapshots_dir: Path,
-    grant_ids: Optional[list] = None
-) -> int:
-    """
-    Generate layman, technical, and executive summaries for grants.
-
-    Args:
-        snapshots_dir: Directory containing grant JSON files
-        grant_ids: Optional list of grant IDs to process (defaults to all)
-
-    Returns:
-        Number of grants summarized
-    """
-    import json
-    import asyncio
-
-    try:
-        from ...database import SummaryStore
-        from ...analyzer.config import load_config
-        from ...analyzer.llm_client import LLMClient
-        from ...analyzer.summarizer_optimized import extract_minimal_context
-    except ImportError as e:
-        logger.error(f"Required modules not available: {e}")
-        return 0
-
-    if not snapshots_dir.exists():
-        return 0
-
-    # Initialize MongoDB and LLM
-    try:
-        summary_store = SummaryStore()
-        config = load_config()
-        # Override to use gpt-5-mini for overnight batch processing
-        config.model = "gpt-5-mini"
-        llm_client = LLMClient(config)
-    except Exception as e:
-        logger.error(f"Failed to initialize summary generation: {e}")
-        return 0
-
-    # Load grants to summarize
-    grants_to_process = []
-    for json_file in snapshots_dir.glob("*.json"):
-        try:
-            with open(json_file, "r", encoding="utf-8") as f:
-                grant = json.load(f)
-
-            grant_id = grant.get("id") or json_file.stem
-
-            # Filter by grant_ids if provided
-            if grant_ids and grant_id not in grant_ids:
-                continue
-
-            grants_to_process.append(grant)
-        except Exception as e:
-            logger.warning(f"Failed to read {json_file}: {e}")
-
-    if not grants_to_process:
-        logger.info("No grants to summarize")
-        return 0
-
-    logger.info(f"Generating summaries for {len(grants_to_process)} grants...")
-
-    # Generate summaries in parallel batches with bulk write
-    async def generate_all_summaries():
-        tasks = []
-        for grant in grants_to_process:
-            tasks.append(generate_grant_summaries(grant, llm_client))
-
-        # Process in parallel (batches of 10)
-        all_summaries = []
-        for i in range(0, len(tasks), 10):
-            batch = tasks[i:i+10]
-            batch_results = await asyncio.gather(*batch, return_exceptions=True)
-
-            # Collect all successful summaries for bulk write
-            for result in batch_results:
-                if isinstance(result, list):
-                    all_summaries.extend(result)
-
-        # Bulk write all summaries at once (more efficient)
-        if all_summaries:
-            saved_count = summary_store.bulk_save_summaries(all_summaries)
-            logger.info(f"Bulk saved {saved_count} summaries")
-            return saved_count // 3  # Divide by 3 since we generate 3 types per grant
-
-        return 0
-
-    async def generate_grant_summaries(grant, client):
-        """Generate all 3 summary types for a single grant (returns list of summaries)."""
-        grant_id = grant.get("id", "unknown")
-
-        try:
-            # Extract minimal context
-            context = extract_minimal_context(grant)
-
-            # Generate 3 summary types in parallel
-            summary_types = [
-                ("layman", "Explain this grant in simple, everyday language that anyone can understand."),
-                ("technical", "Provide a detailed technical summary of this grant, including eligibility criteria and funding details."),
-                ("exec", "Provide a concise executive summary highlighting key points and deadlines.")
-            ]
-
-            async def generate_typed_summary(summary_type, instruction):
-                prompt = f"{instruction}\n\n{context}"
-
-                try:
-                    # Use streaming=False for batch processing
-                    summary = client.summarize(prompt, max_tokens=300)
-                    return {
-                        "grant_id": grant_id,
-                        "summary_type": summary_type,
-                        "summary_text": summary,
-                        "metadata": {"model": client.model, "context_length": len(context)}
-                    }
-                except Exception as e:
-                    logger.error(f"Failed to generate {summary_type} summary for {grant_id}: {e}")
-                    return None
-
-            # Generate all 3 types in parallel
-            results = await asyncio.gather(*[
-                generate_typed_summary(stype, instruction)
-                for stype, instruction in summary_types
-            ], return_exceptions=True)
-
-            # Filter out None results
-            return [r for r in results if r is not None and isinstance(r, dict)]
-
-        except Exception as e:
-            logger.error(f"Failed to process grant {grant_id}: {e}")
-            return []
-
-    try:
-        summarized_count = asyncio.run(generate_all_summaries())
-        logger.info(f"Successfully generated summaries for {summarized_count} grants")
-        return summarized_count
-    except Exception as e:
-        logger.error(f"Summary generation failed: {e}", exc_info=True)
-        return 0
-
-
 def run_crawl_cycle() -> CrawlResult:
     """
     Run a complete crawl and maintenance cycle.
@@ -353,7 +212,6 @@ def run_crawl_cycle() -> CrawlResult:
     try:
         # Step 1: Discover and fetch new grants from Innovate UK
         new_grants = 0
-        new_grant_ids = []
         try:
             from ...crawler.discover_grants import discover_and_fetch_grants
 
@@ -362,7 +220,6 @@ def run_crawl_cycle() -> CrawlResult:
                 discover_and_fetch_grants(SNAPSHOTS_DIR, skip_existing=True)
             )
             new_grants = newly_fetched
-            new_grant_ids = [f.replace(".json", "") for f in new_files]
             logger.info(
                 f"Grant discovery: {total_discovered} total, "
                 f"{newly_fetched} newly fetched"
@@ -373,32 +230,19 @@ def run_crawl_cycle() -> CrawlResult:
             logger.error(f"Grant discovery failed: {e}", exc_info=True)
             # Continue with other steps even if discovery fails
 
-        # Step 2: Generate summaries for new grants (using gpt-5-mini)
-        summaries_generated = 0
-        if new_grant_ids:
-            try:
-                logger.info(f"Generating layman summaries for {len(new_grant_ids)} new grants...")
-                summaries_generated = generate_summaries_for_grants(
-                    SNAPSHOTS_DIR,
-                    grant_ids=new_grant_ids
-                )
-                logger.info(f"Generated summaries for {summaries_generated} grants")
-            except Exception as e:
-                logger.error(f"Summary generation failed: {e}", exc_info=True)
-
-        # Step 3: Check for duplicates
+        # Step 2: Check for duplicates
         duplicates = deduplicate_grants(SNAPSHOTS_DIR)
 
-        # Step 4: Mark closed grants
+        # Step 3: Mark closed grants
         closed = mark_closed_grants(SNAPSHOTS_DIR)
 
-        # Step 5: Rebuild index
+        # Step 4: Rebuild index
         index_rebuilt = rebuild_search_index()
 
         result = CrawlResult(
             timestamp=start_time,
             new_grants=new_grants,
-            updated_grants=summaries_generated,  # Track summaries in updated_grants
+            updated_grants=0,
             closed_grants=closed,
             duplicates_removed=duplicates,
             index_rebuilt=index_rebuilt,

analyzer/crawler/snapshot.py

@@ -1,626 +0,0 @@
-import json
-import re
-import pathlib
-import asyncio
-from urllib.parse import urlparse
-from datetime import datetime, UTC
-from typing import List, Optional, Tuple, Dict
-from playwright.async_api import async_playwright
-from bs4 import BeautifulSoup, Tag, NavigableString
-import typer
-
-# ---------------------------------------------------------------------------
-# HELPERS
-# ---------------------------------------------------------------------------
-
-EXPECTED_TITLES = [
-    "Summary",
-    "Eligibility",
-    "Scope",
-    "Dates",
-    "How to apply",
-    "Supporting information",
-]
-
-# Accept close-enough labels and alias them to canonical 6
-SECTION_ALIASES = {
-    "who can apply": "Eligibility",
-    "who’s eligible": "Eligibility",
-    "who is eligible": "Eligibility",
-    "applicant eligibility": "Eligibility",
-    "what we ask you": "How to apply",
-    "apply": "How to apply",
-    "application process": "How to apply",
-    "supporting info": "Supporting information",
-    "key dates": "Dates",
-    "timeline": "Dates",
-    "competition dates": "Dates",
-    "overview": "Summary",
-    "summary": "Summary",
-    "scope": "Scope",
-}
-
-def canonical_label(label: str) -> Optional[str]:
-    l = label.strip().lower()
-    for t in EXPECTED_TITLES:
-        if l == t.lower():
-            return t
-    return SECTION_ALIASES.get(l, None)
-
-def norm_key(label: str) -> str:
-    return re.sub(r"[^\w\s-]", "", label).strip().lower().replace(" ", "_") + "_raw"
-
-def parse_deeplink(url: str):
-    u = urlparse(url)
-    m = re.search(r"/competition/(\d+)/overview/([0-9a-f-]{8,})", u.path, re.I)
-    if not m:
-        raise ValueError("Expected: .../competition/{id}/overview/{uuid}")
-    return f"{u.scheme}://{u.netloc}", m.group(1), m.group(2)
-
-def get_competition_nav_anchors(soup: BeautifulSoup) -> List[tuple[str, str]]:
-    anchors: List[tuple[str, str]] = []
-
-    headings = soup.find_all(["h2", "h3", "h4"], string=lambda s: isinstance(s, str) and "competition sections" in s.lower())
-    nav_root: Optional[Tag] = None
-    for h in headings:
-        for sib in h.next_siblings:
-            if isinstance(sib, Tag) and sib.name in ("nav", "ul", "ol", "div"):
-                nav_root = sib
-                break
-        if nav_root:
-            break
-
-    if not nav_root:
-        for candidate in soup.find_all("nav"):
-            if candidate.find("a", href=True):
-                nav_root = candidate
-                break
-
-    if nav_root:
-        seen = set()
-        for a in nav_root.find_all("a", href=True):
-            href = a.get("href", "")
-            if not href.startswith("#"):
-                continue
-            frag = href[1:].strip()
-            raw = (a.get_text(" ", strip=True) or "").strip()
-            if not frag or not raw:
-                continue
-            canon = canonical_label(raw) or raw
-            if canon in EXPECTED_TITLES and frag not in seen:
-                anchors.append((canon, frag))
-                seen.add(frag)
-
-    if not anchors:
-        anchors = [
-            ("Summary", "summary"),
-            ("Eligibility", "eligibility"),
-            ("Scope", "scope"),
-            ("Dates", "dates"),
-            ("How to apply", "how-to-apply"),
-            ("Supporting information", "supporting-information"),
-        ]
-    return anchors
-
-# -----------------------------
-# Footer / cookie / consent trimmer
-# -----------------------------
-
-_FOOTER_STOPS = [
-    "Need help with this service?",
-    "Support links",
-    "GOV.UK uses cookies",
-    "Create one update function for each consent parameter",
-    "© Crown copyright",
-    "All content is available under the Open Government Licence",
-]
-
-def trim_footer(text: str) -> str:
-    if not text:
-        return text
-    for marker in _FOOTER_STOPS:
-        i = text.find(marker)
-        if i != -1:
-            return text[:i].rstrip()
-    return text
-
-def _strip_boilerplate(soup: BeautifulSoup):
-    selectors = [
-        "#global-cookie-message", ".cookie-banner", "#ccc-notify", "#onetrust-banner-sdk",
-        "footer", ".govuk-footer",
-        ".govuk-prototype-kit-warning",
-    ]
-    for sel in selectors:
-        for el in soup.select(sel):
-            el.decompose()
-
-# ---------------------------------------------------------------------------
-# TITLE
-# ---------------------------------------------------------------------------
-
-def clean_title(raw: str) -> str:
-    if not raw:
-        return raw
-    raw = raw.strip()
-    return re.sub(r"^\s*Funding competition\s+", "", raw, flags=re.I).strip()
-
-def extract_between_ids(soup: BeautifulSoup, start_id: str, end_id: Optional[str]) -> str:
-    start = soup.find(id=start_id)
-    if not start:
-        return ""
-    out_chunks: List[str] = []
-    for el in start.next_elements:
-        if isinstance(el, Tag):
-            if end_id and el.get("id") == end_id:
-                break
-            if el.name in ("script", "style", "noscript"):
-                continue
-        if isinstance(el, NavigableString):
-            txt = el.strip()
-            if txt:
-                out_chunks.append(txt)
-    text = " ".join(out_chunks)
-    text = re.sub(r"\s+", " ", text).strip()
-    text = trim_footer(text)
-    return text
-
-# ---------------------------------------------------------------------------
-# PARSING
-# ---------------------------------------------------------------------------
-
-def slice_by_anchors(html: str) -> dict:
-    soup = BeautifulSoup(html, "html.parser")
-    _strip_boilerplate(soup)
-    anchors = get_competition_nav_anchors(soup)
-    ids_in_order = [aid for _, aid in anchors]
-    id_to_next = {ids_in_order[i]: (ids_in_order[i + 1] if i + 1 < len(ids_in_order) else None)
-                  for i in range(len(ids_in_order))}
-    out = {}
-    for label, start_id in anchors:
-        next_id = id_to_next.get(start_id)
-        key = norm_key(label)
-        out[key] = extract_between_ids(soup, start_id, next_id)
-    return out
-
-# ---------------------------------------------------------------------------
-# MAIN SCRAPER
-# ---------------------------------------------------------------------------
-
-async def fetch_sections_from_overview(url: str) -> tuple[str, dict]:
-    scheme_host, comp_id, uuid = parse_deeplink(url)
-    overview_url = f"{scheme_host}/competition/{comp_id}/overview/{uuid}"
-
-    async with async_playwright() as pw:
-        browser = await pw.chromium.launch(headless=True, args=["--disable-dev-shm-usage"])
-        context = await browser.new_context(
-            user_agent=("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-                        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
-            locale="en-GB",
-            timezone_id="Europe/London",
-        )
-        page = await context.new_page()
-        page.set_default_timeout(30000)
-
-        for attempt in range(2):
-            try:
-                await page.goto(overview_url, wait_until="domcontentloaded")
-                await page.wait_for_load_state("networkidle")
-                break
-            except Exception:
-                if attempt == 1:
-                    raise
-        html = await page.content()
-        await context.close()
-        await browser.close()
-
-    return html, slice_by_anchors(html)
-
-# ---------------------------------------------------------------------------
-# DATE PARSING — SINGLE-LINE CHUNKS
-# ---------------------------------------------------------------------------
-
-# tokens
-_DATE_WORD = r"(?:\d{1,2}\s+[A-Za-z]{3,9}\s+\d{4}|[A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4})"
-_TIME_WORD = r"(?:\d{1,2}:\d{2}\s*[ap]m|\d{1,2}\s*[ap]m)"
-
-_DATE_ONLY_RX = re.compile(_DATE_WORD, re.I)
-_TIME_RX = re.compile(_TIME_WORD, re.I)
-
-# e.g. "9 to 20 March 2026", "9–20 March 2026", "9 - 20 March 2026"
-_DATE_RANGE_RX = re.compile(r"(\d{1,2})\s*(?:to|-|–)\s*(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", re.I)
-
-_MONTHS = {m.lower(): i for i, m in enumerate(
-    ["January","February","March","April","May","June","July","August","September","October","November","December"], 1
-)}
-
-_EXCLUDE_SENTENCE_CUES = ["briefing event", "briefing", "webinar", "register to attend", "register", "info session"]
-
-_LABEL_RULES = [
-    ("opens", ["competition opens", "opens"]),
-    ("closes", ["competition closes", "closes", "deadline"]),
-    ("notify", ["applicants notified", "applicants will be notified", "notification"]),
-    ("project_start", ["project start from", "project starts from", "project start date", "project start"]),
-    ("assessment", ["interview", "assessment", "panel"]),
-    ("results", ["results published", "winners announced"]),
-    ("eligibility_cutoff", ["eligibility closes", "registration closes"]),
-    ("info_session", ["briefing", "webinar", "register"]),
-]
-
-def _classify_label(sent_lower: str) -> str:
-    for norm, cues in _LABEL_RULES:
-        if any(c in sent_lower for c in cues):
-            return norm
-    return "other"
-
-def _parse_single_date(token: str, time_hint: Optional[str]) -> Optional[str]:
-    token = token.strip()
-    m_comma = re.match(r"([A-Za-z]{3,9})\s+(\d{1,2}),?\s+(\d{4})", token)
-    if m_comma:
-        month_name, day, year = m_comma.groups()
-    else:
-        m = re.match(r"(\d{1,2})\s+([A-Za-z]{3,9})\s+(\d{4})", token)
-        if not m:
-            return None
-        day, month_name, year = m.groups()
-    month = _MONTHS.get(month_name.lower())
-    if not month:
-        return None
-    if time_hint:
-        t = time_hint.lower().replace(" ", "")
-        mm = re.match(r"(\d{1,2})(?::(\d{2}))?([ap]m)", t)
-        if mm:
-            hh = int(mm.group(1))
-            mins = int(mm.group(2) or 0)
-            ampm = mm.group(3)
-            if ampm == "pm" and hh != 12: hh += 12
-            if ampm == "am" and hh == 12: hh = 0
-            return f"{int(year):04d}-{month:02d}-{int(day):02d}T{hh:02d}:{mins:02d}:00"
-    return f"{int(year):04d}-{month:02d}-{int(day):02d}"
-
-def parse_dates_singleline(text: str) -> List[dict]:
-    """
-    Split the Dates section into *one milestone per line/chunk*:
-      chunk := from each DATE token up to the next DATE token (or end).
-    Keeps the entire chunk in label_raw.
-    """
-    milestones: List[dict] = []
-    if not text:
-        return milestones
-
-    # find all date token positions
-    matches = list(_DATE_ONLY_RX.finditer(text))
-    if not matches:
-        return milestones
-
-    spans = []
-    for i, m in enumerate(matches):
-        start = m.start()
-        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
-        spans.append((start, end))
-
-    for (start, end) in spans:
-        chunk = text[start:end].strip()
-        if not chunk:
-            continue
-        low = chunk.lower()
-        excluded = any(k in low for k in _EXCLUDE_SENTENCE_CUES)
-
-        # primary date + optional time in this chunk
-        first_date = _DATE_ONLY_RX.search(chunk)
-        time_hint_match = _TIME_RX.search(chunk)
-        iso = _parse_single_date(first_date.group(0), time_hint_match.group(0) if time_hint_match else None) if first_date else None
-
-        # optional same-month day range inside the chunk
-        r = _DATE_RANGE_RX.search(chunk)
-        date_end_iso = None
-        if r:
-            d1, d2, mon_name, year = r.groups()
-            month = _MONTHS.get(mon_name.lower())
-            if month:
-                date_end_iso = f"{int(year):04d}-{month:02d}-{int(d2):02d}"
-                # if the start of the range equals first_date, keep iso as start;
-                # otherwise we still keep iso from first_date (which begins the chunk)
-
-        if iso:
-            milestones.append({
-                "label_raw": chunk,
-                "label_norm": _classify_label(low),
-                "date_iso": iso,
-                "date_iso_end": date_end_iso,
-                "has_time": bool(time_hint_match),
-                "excluded_from_open_close": excluded,
-            })
-
-    return milestones
-
-def pick_open_close_from_milestones(milestones: List[dict]) -> Tuple[Optional[str], Optional[str]]:
-    open_iso = close_iso = None
-    for m in milestones:
-        if m["excluded_from_open_close"]:
-            continue
-        if m["label_norm"] == "opens" and open_iso is None:
-            open_iso = m["date_iso"].split("T")[0]
-        if m["label_norm"] == "closes" and close_iso is None:
-            close_iso = m["date_iso"]
-    if open_iso is None:
-        for m in milestones:
-            if m["label_norm"] == "opens":
-                open_iso = m["date_iso"].split("T")[0]
-                break
-    if close_iso is None:
-        for m in milestones:
-            if m["label_norm"] == "closes":
-                close_iso = m["date_iso"]
-                break
-    return open_iso, close_iso
-
-def derive_aux_dates(milestones: List[dict]) -> Tuple[Optional[str], Optional[str]]:
-    notify = project_start_from = None
-    for m in milestones:
-        if notify is None and m["label_norm"] == "notify":
-            notify = m["date_iso"].split("T")[0]
-        if project_start_from is None and m["label_norm"] == "project_start":
-            project_start_from = m["date_iso"].split("T")[0]
-        if notify and project_start_from:
-            break
-    return notify, project_start_from
-
-# ---------------------------------------------------------------------------
-# FUNDING / COMPENSATION PARSING
-# ---------------------------------------------------------------------------
-
-_MONEY_TOKEN = re.compile(r"(£|\bGBP\s*)([\d,]+(?:\.\d+)?)(?:\s*(million|m|billion|bn|k))?", re.I)
-
-def _money_to_int(sign: str, num_str: str, mag: Optional[str]) -> int:
-    val = float(num_str.replace(",", ""))
-    if mag:
-        m = mag.lower()
-        if m in ("million", "m"):
-            val *= 1_000_000
-        elif m in ("billion", "bn"):
-            val *= 1_000_000_000
-        elif m in ("k",):
-            val *= 1_000
-    return int(round(val))
-
-_TOTAL_CUES = [
-    "total prize fund", "total prize pot", "total funding available", "available in total",
-    "total pot", "prize fund", "funding pot", "overall budget", "total budget",
-    "total allocation", "in total across", "total amount available",
-]
-_AWARD_CUES = [
-    "per project", "each project", "you can apply for", "can apply for", "apply for up to",
-    "grant of up to", "awards of up to", "awards between", "awards of between",
-    "fund between", "we will fund", "we can fund", "project costs between",
-    "total eligible project costs between", "your project must have total costs between",
-    "maximum grant", "minimum grant", "maximum funding", "minimum funding",
-    "up to", "no more than", "at least",
-    "grant funding request", "eligible grant funding", "eligible grant", "funding request must be between",
-]
-_EXCLUDE_CUES = [
-    "market", "industry", "global", "worldwide", "valuation", "addressable", "gdp",
-    "economy", "sector value", "turnover", "revenue", "jobs", "headcount",
-]
-
-_RANGE_PATTERNS = [
-    re.compile(rf"(?:between|from)\s+{_MONEY_TOKEN.pattern}\s+(?:and|to)\s+{_MONEY_TOKEN.pattern}", re.I),
-    re.compile(rf"{_MONEY_TOKEN.pattern}\s*(?:to|-)\s*{_MONEY_TOKEN.pattern}", re.I),
-]
-_MAX_PATTERNS = [
-    re.compile(rf"(?:up to|no more than|max(?:imum)?(?:\s+grant|\s+funding)?)\s+{_MONEY_TOKEN.pattern}", re.I),
-]
-_MIN_PATTERNS = [
-    re.compile(rf"(?:at least|minimum(?:\s+grant|\s+funding)?)\s+{_MONEY_TOKEN.pattern}", re.I),
-]
-
-def _contains_any(text: str, cues: List[str]) -> bool:
-    low = text.lower()
-    return any(c in low for c in cues)
-
-def _is_excluded_sentence(sent: str) -> bool:
-    return _contains_any(sent, _EXCLUDE_CUES)
-
-def _split_sentences_generic(text: str) -> List[str]:
-    parts = re.split(r"(?:\n+|(?<=[\.\!\?])\s+)", text)
-    return [p.strip() for p in parts if p and p.strip()]
-
-def _find_total_pot(text: str) -> Optional[int]:
-    if not text:
-        return None
-    best = None
-    for sent in _split_sentences_generic(text):
-        if _is_excluded_sentence(sent):
-            continue
-        if _contains_any(sent, _TOTAL_CUES):
-            vals = []
-            for m in _MONEY_TOKEN.finditer(sent):
-                _, num_str, mag = m.groups()
-                vals.append(_money_to_int("£", num_str, mag))
-            if vals:
-                v = max(vals)
-                best = v if best is None or v > best else best
-    return best
-
-def _find_award_range(text: str) -> Tuple[Optional[int], Optional[int]]:
-    if not text:
-        return None, None
-
-    # Strong: explicit ranges in a sentence that has award cues
-    for sent in _split_sentences_generic(text):
-        if _is_excluded_sentence(sent):
-            continue
-        if not _contains_any(sent, _AWARD_CUES):
-            continue
-        for rx in _RANGE_PATTERNS:
-            m = rx.search(sent)
-            if not m:
-                continue
-            monies = list(_MONEY_TOKEN.finditer(m.group(0)))
-            if len(monies) >= 2:
-                v1 = _money_to_int(*("£", monies[-2].group(2), monies[-2].group(3)))
-                v2 = _money_to_int(*("£", monies[-1].group(2), monies[-1].group(3)))
-                lo, hi = sorted([v1, v2])
-                return lo, hi
-
-    # Next: max-only / min-only with cues
-    chosen_min = None
-    chosen_max = None
-    for sent in _split_sentences_generic(text):
-        if _is_excluded_sentence(sent):
-            continue
-        if not _contains_any(sent, _AWARD_CUES):
-            continue
-
-        if chosen_max is None:
-            for rx in _MAX_PATTERNS:
-                m = rx.search(sent)
-                if m:
-                    money = _MONEY_TOKEN.search(m.group(0))
-                    if money:
-                        chosen_max = _money_to_int(*("£", money.group(2), money.group(3)))
-                        break
-
-        if chosen_min is None:
-            for rx in _MIN_PATTERNS:
-                m = rx.search(sent)
-                if m:
-                    money = _MONEY_TOKEN.search(m.group(0))
-                    if money:
-                        chosen_min = _money_to_int(*("£", money.group(2), money.group(3)))
-                        break
-
-        if chosen_min is not None and chosen_max is not None:
-            break
-
-    return chosen_min, chosen_max
-
-# NEW: funding rates & duration
-_RATE_LINE = re.compile(
-    r"up to\s*(\d{1,3})%\s*if you are a\s*(?:micro|small).*?up to\s*(\d{1,3})%\s*if you are a\s*medium.*?up to\s*(\d{1,3})%\s*if you are a\s*large",
-    re.I | re.S,
-)
-def _find_funding_rates(text: str) -> Optional[dict]:
-    if not text:
-        return None
-    m = _RATE_LINE.search(text)
-    if not m:
-        return None
-    small, medium, large = map(int, m.groups())
-    return {"micro_small": small, "medium": medium, "large": large}
-
-_DURATION_RX = re.compile(r"last\s+between\s+(\d{1,3})\s*(?:and|to|–|-)\s*(\d{1,3})\s+months", re.I)
-def _find_duration_months(text: str) -> Tuple[Optional[int], Optional[int]]:
-    if not text:
-        return None, None
-    m = _DURATION_RX.search(text)
-    if not m:
-        return None, None
-    lo, hi = map(int, m.groups())
-    return (lo if lo <= hi else hi), (hi if hi >= lo else lo)
-
-def extract_funding(sections: dict) -> dict:
-    summary = sections.get("summary_raw", "") or ""
-    support = sections.get("supporting_information_raw", "") or ""
-    scope = sections.get("scope_raw", "") or ""
-    eligibility = sections.get("eligibility_raw", "") or ""
-    all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
-
-    total_pot = _find_total_pot(summary) or _find_total_pot(support) or _find_total_pot(all_text)
-
-    min_award = max_award = None
-    for candidate in (summary, eligibility, support, scope, all_text):
-        lo, hi = _find_award_range(candidate)
-        if lo is not None or hi is not None:
-            if lo is not None: min_award = lo
-            if hi is not None: max_award = hi
-            break
-
-    rates = None
-    for candidate in (eligibility, support, all_text):
-        rates = _find_funding_rates(candidate or "")
-        if rates:
-            break
-
-    return {"min": min_award, "max": max_award, "total_pot": total_pot, "rates": rates}
-
-# ---------------------------------------------------------------------------
-# MAIN
-# ---------------------------------------------------------------------------
-
-async def _main_async(url: str):
-    m = re.search(r"/competition/(\d+)", url)
-    slug = f"competition-{m.group(1)}" if m else re.sub(r"[^a-z0-9]+", "-", url.lower()).strip("-")[-60:]
-    out_path = pathlib.Path("data/snapshots") / f"{slug}.json"
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-
-    html, sections = await fetch_sections_from_overview(url)
-    soup = BeautifulSoup(html, "html.parser")
-    h1 = soup.find("h1")
-    raw_title = h1.get_text(" ", strip=True) if h1 else ""
-    title = clean_title(raw_title)
-
-    # Dates -> single-line chunks
-    dates_text = sections.get("dates_raw", "") or ""
-    milestones = parse_dates_singleline(dates_text)
-
-    # Derive open/close from milestones (with exclusions for briefing lines)
-    open_date, close_date = pick_open_close_from_milestones(milestones)
-
-    # If either missing, try scanning all text but still as single-line chunks
-    if not open_date or not close_date:
-        all_text = " ".join(v for v in sections.values() if isinstance(v, str) and v)
-        extra = parse_dates_singleline(all_text)
-        # merge de-duped by (label_raw, date_iso, date_iso_end)
-        seen = {(m["label_raw"], m["date_iso"], m.get("date_iso_end")) for m in milestones}
-        for m2 in extra:
-            key = (m2["label_raw"], m2["date_iso"], m2.get("date_iso_end"))
-            if key not in seen:
-                milestones.append(m2)
-                seen.add(key)
-        od2, cd2 = pick_open_close_from_milestones(milestones)
-        open_date = open_date or od2
-        close_date = close_date or cd2
-
-    notify_date, project_start_from = derive_aux_dates(milestones)
-
-    # Funding
-    funding = extract_funding(sections)
-
-    # Duration months
-    dur_min, dur_max = _find_duration_months(sections.get("eligibility_raw", "") or "")
-    if dur_min is None or dur_max is None:
-        all_text_for_duration = " ".join(v for v in sections.values() if isinstance(v, str) and v)
-        dur_min, dur_max = _find_duration_months(all_text_for_duration)
-    duration_months = {"min": dur_min, "max": dur_max}
-
-    snapshot = {
-        "url": url,
-        "title": title,
-        "programme": "",
-        "round": "",
-        "open_date": open_date,
-        "close_date": close_date,
-        "notify_date": notify_date,
-        "project_start_from": project_start_from,
-        "funding": funding,
-        "duration_months": duration_months,
-        "sections": sections,
-        "pdfs": [],
-        "summaries": {},
-        "extracted": {"milestones": milestones},
-        "wonky": {"score": 0.0, "reasons": []},
-        "prev_round_refs": [],
-        "diff_summary": "",
-        "history_stats": {},
-        "created_at": datetime.now(UTC).isoformat(),
-        "updated_at": datetime.now(UTC).isoformat(),
-    }
-
-    out_path.write_text(json.dumps(snapshot, indent=2), encoding="utf-8")
-    print(f"Snapshot saved to {out_path}")
-
-def main(url: str = typer.Argument(..., help="IFS overview URL e.g. .../competition/{id}/overview/{uuid}")):
-    asyncio.run(_main_async(url))
-
-if __name__ == "__main__":
-    typer.run(main)

analyzer/data_loader.py

@@ -117,14 +117,61 @@ def _load_past_winners_from_json_dir(json_dir: Path) -> List[Dict[str, Any]]:
     return records
 
 
-def load_past_winners(history_xlsx: Path | str | None = None,
-                      history_json_dir: Path | str | None = None) -> List[Dict[str, Any]]:
-    """Load past winners from either Excel (preferred) or a JSON folder.
+def _load_past_winners_from_jsonl(jsonl_path: Path) -> List[Dict[str, Any]]:
+    """Load past winners from JSONL file (optionally gzipped)."""
+    import gzip
 
-    Priority order: JSON dir (if provided) > Excel (if provided).
+    records: List[Dict[str, Any]] = []
+
+    # Check if file is gzipped
+    open_func = gzip.open if str(jsonl_path).endswith('.gz') else open
+
+    try:
+        with open_func(jsonl_path, 'rt', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                    if isinstance(rec, dict):
+                        records.append(rec)
+                except json.JSONDecodeError:
+                    continue
+        return records
+    except Exception as e:
+        logger.warning(f"Failed to load JSONL from {jsonl_path}: {e}")
+        return []
+
+
+def load_past_winners(
+    history_xlsx: Path | str | None = None,
+    history_json_dir: Path | str | None = None,
+    history_jsonl: Path | str | None = None
+) -> List[Dict[str, Any]]:
+    """Load past winners from Excel, JSONL, or JSON directory.
+
+    Priority order: JSONL > JSON dir > Excel.
     If neither exists, returns an empty list.
     """
-    # JSON dir first if present
+    # Try JSONL first (most efficient for large datasets)
+    if history_jsonl is not None:
+        jsonl = Path(history_jsonl)
+        if jsonl.exists():
+            recs = _load_past_winners_from_jsonl(jsonl)
+            if recs:
+                logger.info("Loaded %d past winners from JSONL: %s", len(recs), jsonl)
+                return recs
+
+    # Also check for past_winners.jsonl.gz in default location (for HF deployment)
+    default_jsonl = Path("data/past_winners.jsonl.gz")
+    if default_jsonl.exists() and history_jsonl is None:
+        recs = _load_past_winners_from_jsonl(default_jsonl)
+        if recs:
+            logger.info("Loaded %d past winners from default JSONL: %s", len(recs), default_jsonl)
+            return recs
+
+    # JSON dir next
     if history_json_dir is not None:
         jdir = Path(history_json_dir)
         if jdir.exists():
@@ -135,7 +182,7 @@ def load_past_winners(history_xlsx: Path | str | None = None,
             else:
                 logger.info("No JSON past winners found under %s", jdir)
 
-    # Excel next
+    # Excel last
     if history_xlsx is not None:
         xlsx = Path(history_xlsx)
         if xlsx.exists():

analyzer/llm_client.py

@@ -3,14 +3,10 @@ from __future__ import annotations
 import logging
 import os
 import time
-from typing import Any, Dict, List, Optional, Callable, Literal
+from typing import Any, Dict, List, Optional, Callable
 
 from .utils.errors import LLMError, ConfigError
 
-ModelType = Literal["router", "translator", "analyzer"]
-VerbosityLevel = Literal["low", "medium", "high"]
-ReasoningEffort = Literal["minimal", "medium", "high"]
-
 try:
     from openai import OpenAI
     import httpx
@@ -32,14 +28,9 @@ class LLMClient:
     def __init__(self, cfg: Any):
         # Extract config (supports both dict and object)
         self.provider = self._get_cfg(cfg, "provider", "openai")
-        self.model = self._get_cfg(cfg, "model", "gpt-5-mini")
+        self.model = self._get_cfg(cfg, "model", "gpt-4o-mini")
         self.disable_llm = self._get_cfg(cfg, "disable_llm", False)
 
-        # Store model variants for different use cases
-        self.model_router = self._get_cfg(cfg, "model_router", "gpt-5-nano")
-        self.model_translator = self._get_cfg(cfg, "model_translator", "gpt-5-mini")
-        self.model_analyzer = self._get_cfg(cfg, "model_analyzer", "gpt-5")
-
         api_key = self._get_cfg(cfg, "api_key") or os.getenv("OPENAI_API_KEY")
         base_url = self._get_cfg(cfg, "base_url") or os.getenv(
             "OPENAI_API_BASE",
@@ -94,54 +85,6 @@ class LLMClient:
         """Check if LLM client is ready to use."""
         return self.client is not None and not self.disable_llm
 
-    def _get_model_for_type(self, model_type: Optional[ModelType] = None) -> str:
-        """Get the appropriate model for the given type."""
-        if model_type == "router":
-            return self.model_router
-        elif model_type == "translator":
-            return self.model_translator
-        elif model_type == "analyzer":
-            return self.model_analyzer
-        else:
-            return self.model
-
-    @staticmethod
-    def get_recommended_params(
-        task_type: str
-    ) -> Dict[str, Any]:
-        """
-        Get recommended verbosity and reasoning_effort for common tasks.
-
-        Args:
-            task_type: One of "translation", "analysis", "routing", "summary"
-
-        Returns:
-            Dict with verbosity and reasoning_effort settings
-        """
-        presets = {
-            "translation": {
-                "verbosity": "medium",
-                "reasoning_effort": "minimal"
-            },
-            "analysis": {
-                "verbosity": "high",
-                "reasoning_effort": "high"
-            },
-            "routing": {
-                "verbosity": "low",
-                "reasoning_effort": "minimal"
-            },
-            "summary": {
-                "verbosity": "medium",
-                "reasoning_effort": "medium"
-            },
-            "comparison": {
-                "verbosity": "high",
-                "reasoning_effort": "high"
-            }
-        }
-        return presets.get(task_type, {"verbosity": "medium", "reasoning_effort": "medium"})
-
     def _retry_with_backoff(
         self,
         fn: Callable[[], Any],
@@ -200,9 +143,6 @@ class LLMClient:
         temperature: float = 0.2,
         top_p: float = 1.0,
         stream: bool = False,
-        model_type: Optional[ModelType] = None,
-        verbosity: Optional[VerbosityLevel] = None,
-        reasoning_effort: Optional[ReasoningEffort] = None,
     ) -> str:
         """
         Single chat completion call with optional streaming.
@@ -213,20 +153,6 @@ class LLMClient:
             temperature: Sampling temperature (0-2)
             top_p: Nucleus sampling parameter
             stream: If True, returns generator yielding tokens (else full response)
-            model_type: Type of model to use ('router', 'translator', or 'analyzer')
-            verbosity: Response length control (GPT-5 feature)
-                - 'low': Brief, concise responses
-                - 'medium': Standard length responses
-                - 'high': Detailed, comprehensive responses
-            reasoning_effort: Thinking time control (GPT-5 feature)
-                - 'minimal': Quick, straightforward responses
-                - 'medium': Moderate analysis and reasoning
-                - 'high': Deep analysis and careful reasoning
-
-        Recommended combinations:
-            - Translations: verbosity='medium', reasoning_effort='minimal'
-            - Complex analysis: verbosity='high', reasoning_effort='high'
-            - Routing decisions: verbosity='low', reasoning_effort='minimal'
 
         Returns:
             Generated text response (or generator if stream=True)
@@ -242,28 +168,16 @@ class LLMClient:
                 "LLM client not initialized. Check API key and configuration."
             )
 
-        # Select model based on type
-        selected_model = self._get_model_for_type(model_type)
-
         def _make_call():
             try:
-                # Build API call parameters
-                params = {
-                    "model": selected_model,
-                    "messages": messages,
-                    "temperature": temperature,
-                    "top_p": top_p,
-                    "max_tokens": max_tokens,
-                    "stream": stream,
-                }
-
-                # Add GPT-5 specific parameters if provided
-                if verbosity is not None:
-                    params["verbosity"] = verbosity
-                if reasoning_effort is not None:
-                    params["reasoning_effort"] = reasoning_effort
-
-                resp = self.client.chat.completions.create(**params)
+                resp = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    temperature=temperature,
+                    top_p=top_p,
+                    max_tokens=max_tokens,
+                    stream=stream,
+                )
             except Exception as e:
                 # Handle httpx exceptions if available
                 if httpx and isinstance(e, httpx.TimeoutException):
@@ -288,19 +202,6 @@ class LLMClient:
                 if not content:
                     raise LLMError("LLM returned empty response")
 
-                # Record metrics (token usage and model distribution)
-                try:
-                    from src.monitoring import record_tokens, record_model_use
-                    if hasattr(resp, 'usage') and resp.usage:
-                        record_tokens(
-                            prompt_tokens=resp.usage.prompt_tokens,
-                            completion_tokens=resp.usage.completion_tokens
-                        )
-                    record_model_use(selected_model)
-                except Exception as e:
-                    # Don't fail the request if metrics recording fails
-                    logging.warning(f"Failed to record metrics: {e}")
-
                 return content.strip()
 
         return self._retry_with_backoff(_make_call, max_attempts=self.max_retries)
@@ -327,8 +228,6 @@ class LLMClient:
         system_text: str = "You are an expert UK grant analyst. Be precise, grounded, concise.",
         max_tokens: int = 900,
         temperature: float = 0.2,
-        verbosity: Optional[VerbosityLevel] = None,
-        reasoning_effort: Optional[ReasoningEffort] = None,
     ) -> str:
         """
         Convenience: single-turn chat.
@@ -338,8 +237,6 @@ class LLMClient:
             system_text: System prompt
             max_tokens: Maximum tokens
             temperature: Sampling temperature
-            verbosity: Response length control (GPT-5)
-            reasoning_effort: Thinking time control (GPT-5)
 
         Returns:
             Generated summary
@@ -351,13 +248,7 @@ class LLMClient:
             {"role": "system", "content": system_text},
             {"role": "user", "content": user_text},
         ]
-        return self.chat(
-            messages,
-            max_tokens=max_tokens,
-            temperature=temperature,
-            verbosity=verbosity,
-            reasoning_effort=reasoning_effort
-        )
+        return self.chat(messages, max_tokens=max_tokens, temperature=temperature)
 
     def summarize_long(
         self,

analyzer/prompt_templates.py

@@ -14,7 +14,19 @@ from typing import Dict
 _SYSTEM_DEFAULT = (
     "You are an expert grant analyst. Produce crisp, factual executive summaries "
     "for UK innovation funding calls using ONLY the provided context. "
-    "For vague or messy user requests, call the tool `search_grants` with `query` set to the raw user text and include any obvious `filters` you can infer (e.g., status=\"open\", audience=\"SME\", theme like \"battery\", or timeframe). "
+    "\n"
+    "## Tool usage guide:\n"
+    "- For searching CURRENT GRANTS: Use `search_grants` or `list_grants` with queries like 'AI', 'battery', 'net zero', etc.\n"
+    "- For finding PAST WINNERS: Use `search_past_winners` to find previous winners by project name, organization, or grant name.\n"
+    "- If user asks about 'who won' or 'previous winners': ALWAYS use `search_past_winners`.\n"
+    "- If user asks about 'funding opportunities' or 'apply for': ALWAYS use `search_grants` or `list_grants`.\n"
+    "- For vague user requests, call `search_grants` with `query` set to the raw user text and include filters (e.g., status, audience, theme).\n"
+    "\n"
+    "## About grants vs prizes:\n"
+    "- GRANTS: Traditional funding for projects/research (current opportunities in the database).\n"
+    "- PRIZES: Competition-based funding with winners (e.g., 'Agentic AI Pioneers Prize').\n"
+    "- Both can be found! If you find a prize when searching for a grant, note that it's a prize and explain the difference.\n"
+    "\n"
     "Be precise, avoid hype, and NEVER invent facts. If a detail is missing, say so briefly. "
     "Prefer bullet points. Keep to 250–400 words."
 )
@@ -59,8 +71,12 @@ def build_prompt(provider: str, context_text: str, *, style: str = "default") ->
 OPEN_SYSTEM = (
     "You are a UK grant analyst and research copilot.\n"
     "- Prefer grounded answers using the provided context/snippets when available.\n"
-    "- If a detail isn’t in the context, say so briefly or mark it as uncertain.\n"
-    "- Choose the clearest format for the user’s ask (short answer, bullets, table, or brief narrative) — your call.\n"
+    "- If a detail isn't in the context, say so briefly or mark it as uncertain.\n"
+    "- Use tools proactively:\n"
+    "  * `search_grants`/`list_grants` for funding opportunities\n"
+    "  * `search_past_winners` for previous winners and past projects\n"
+    "- Note distinctions: GRANTS are traditional funding, PRIZES are competition-based (e.g., 'Agentic AI Pioneers Prize').\n"
+    "- Choose the clearest format for the user's ask (short answer, bullets, table, or brief narrative) — your call.\n"
     "- Be concise by default; expand only if asked.\n"
     "- Never fabricate URLs or specific numbers not present in context."
 )

analyzer/summarizer_optimized.py

@@ -3,9 +3,9 @@ summarizer_optimized.py — High-performance grant summarization with:
 - PARALLELIZATION: asyncio.gather() for concurrent processing
 - STREAMING: Yield results as batches complete (for async contexts)
 - BATCH PROCESSING: 5 grants per API call via clever prompting
-- SMART CONTEXT: Extract only essential fields (~200 tokens per grant)
+- SMART CONTEXT: Extract only essential fields (~500 tokens per grant)
 - CACHING: In-memory cache with 1-hour TTL (Redis-ready pattern)
-- MODEL OPTIMIZATION: gpt-5-mini for basic summaries (fast and cost-effective)
+- MODEL OPTIMIZATION: gpt-3.5-turbo for basic summaries (10x cheaper, 2x faster)
 
 Performance target: <30 seconds for 30 grants (vs. 7 minutes sequential)
 
@@ -31,14 +31,6 @@ from .llm_client import LLMClient
 
 logger = logging.getLogger(__name__)
 
-# Import tiktoken for token counting
-try:
-    import tiktoken
-    HAS_TIKTOKEN = True
-except ImportError:
-    HAS_TIKTOKEN = False
-    logger.warning("tiktoken not available - token counting disabled")
-
 
 # ================================= CACHING LAYER =================================
 
@@ -86,55 +78,21 @@ class SummaryCache:
 
 # ================================= CONTEXT EXTRACTION =================================
 
-def _get_first_sentences(text: str, n: int = 3) -> str:
-    """Extract first N sentences from text."""
-    if not text:
-        return ""
-    sentences = text.split('. ')
-    return '. '.join(sentences[:n]).strip() + ('.' if len(sentences) > n else '')
-
-
-def _count_tokens(text: str) -> int:
-    """Count tokens in text using tiktoken (if available)."""
-    if not HAS_TIKTOKEN:
-        # Rough approximation: 1 token ≈ 4 characters
-        return len(text) // 4
-
-    try:
-        # Use o200k_base encoding for GPT-5 models (fallback to cl100k_base for GPT-4)
-        try:
-            encoding = tiktoken.get_encoding("o200k_base")
-        except:
-            encoding = tiktoken.get_encoding("cl100k_base")
-        return len(encoding.encode(text))
-    except Exception:
-        # Fallback to character approximation
-        return len(text) // 4
-
-
 def extract_minimal_context(grant: Dict[str, Any], past_winners: Optional[List[Dict[str, Any]]] = None) -> str:
     """
-    Extract only essential fields (~200 tokens per grant).
-
-    Aggressive reduction strategy:
-    - Title: max 100 chars
-    - Deadline: as-is
-    - Funding: as-is
-    - Summary: first 3 sentences only
-    - Eligibility: first 2 sentences only
-    - NO past winners (saves ~50-100 tokens)
+    Extract only essential fields (~500 tokens per grant).
 
-    Target: <200 tokens per grant for efficient batch processing
+    Instead of full HTML, extract:
+    - title, deadline, max_funding, brief description (first 200 words), eligibility
     """
     parts = []
 
-    # Title (truncate to 100 chars)
+    # Title
     title = grant.get("title") or grant.get("name") or grant.get("competition_title") or "(untitled)"
-    title = title[:100]
     parts.append(f"TITLE: {title}")
 
     # Deadline
-    deadline = grant.get("deadline") or grant.get("close_date")
+    deadline = grant.get("deadline")
     if deadline:
         parts.append(f"DEADLINE: {deadline}")
 
@@ -143,7 +101,7 @@ def extract_minimal_context(grant: Dict[str, Any], past_winners: Optional[List[D
     if funding:
         parts.append(f"FUNDING: {funding}")
 
-    # Summary/Description (first 3 sentences only)
+    # Brief summary/description (first 200 words)
     summary_raw = None
     for field in ["summary_raw", "summary", "description", "overview"]:
         if grant.get("sections", {}).get(field):
@@ -151,14 +109,11 @@ def extract_minimal_context(grant: Dict[str, Any], past_winners: Optional[List[D
             break
 
     if summary_raw:
-        # Extract first 3 sentences
-        summary_short = _get_first_sentences(summary_raw, 3)
-        # Further truncate to 200 chars if needed
-        if len(summary_short) > 200:
-            summary_short = summary_short[:200] + "..."
-        parts.append(f"SUMMARY: {summary_short}")
-
-    # Eligibility (first 2 sentences only)
+        # Truncate to ~200 words
+        words = summary_raw.split()[:200]
+        parts.append(f"DESCRIPTION: {' '.join(words)}")
+
+    # Eligibility
     eligibility = None
     for field in ["eligibility_raw", "eligibility", "who_can_apply"]:
         if grant.get("sections", {}).get(field):
@@ -166,24 +121,19 @@ def extract_minimal_context(grant: Dict[str, Any], past_winners: Optional[List[D
             break
 
     if eligibility:
-        # Extract first 2 sentences
-        eligibility_short = _get_first_sentences(eligibility, 2)
-        # Further truncate to 150 chars if needed
-        if len(eligibility_short) > 150:
-            eligibility_short = eligibility_short[:150] + "..."
-        parts.append(f"ELIGIBILITY: {eligibility_short}")
-
-    # Build final context
-    context = "\n".join(parts)
-
-    # Token count check (for logging)
-    token_count = _count_tokens(context)
-    if token_count > 200:
-        logger.debug(
-            f"Context for '{title[:30]}...' is {token_count} tokens (target: 200)"
-        )
-
-    return context
+        words = eligibility.split()[:150]
+        parts.append(f"ELIGIBILITY: {' '.join(words)}")
+
+    # Past winners (if provided)
+    if past_winners:
+        parts.append(f"\nPAST WINNERS ({len(past_winners)} records):")
+        for winner in past_winners[:3]:  # Only first 3 to save tokens
+            org = winner.get("lead_org", "Unknown")
+            amount = winner.get("award_amount", "Unknown")
+            title_w = winner.get("project_title", "Unknown")
+            parts.append(f"  • {org}: {title_w} ({amount})")
+
+    return "\n".join(parts)
 
 
 # ================================= BATCH SUMMARIZATION =================================

analyzer/telemetry/logger.py

@@ -5,7 +5,7 @@ Usage:
     from analyzer.telemetry.logger import QALogger
     log = QALogger("logs/chat.jsonl")
     log.write(user="find farming grants", intent="search", args={"keyword":"farming"},
-              answer_md="...", ok=True, latency_ms=321, meta={"model":"gpt-5-mini"})
+              answer_md="...", ok=True, latency_ms=321, meta={"model":"gpt-4.1-mini"})
 """
 from __future__ import annotations
 from dataclasses import asdict, dataclass, field

analyzer/utils/query_logger.py

@@ -81,7 +81,7 @@ class QueryLogger:
             success: Whether the interaction was successful
             rating: Optional 1-5 rating (for RLHF)
             feedback: Optional text feedback (for RLHF)
-            model: Model name used (e.g., "gpt-5-mini")
+            model: Model name used (e.g., "gpt-4-mini")
             tokens_used: Number of tokens consumed
             metadata: Additional metadata to log
         """
@@ -305,7 +305,7 @@ if __name__ == "__main__":
         success=True,
         rating=5,
         feedback="Very helpful!",
-        model="gpt-5-mini"
+        model="gpt-4-mini"
     )
 
     # Get stats

analyzer/utils/text.py

@@ -34,7 +34,7 @@ def safe_truncate_chars(s: str, n: int) -> str:
         return s
     return s[: max(0, n - 1)] + "…"
 
-def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-5-mini") -> str:
+def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-4o-mini") -> str:
     """
     Best effort token truncation. If tiktoken is available, use it;
     otherwise approximate by ~4 chars/token heuristic.
@@ -42,11 +42,7 @@ def safe_truncate_tokens(s: str, max_tokens: int, *, model: str = "gpt-5-mini")
     s = s or ""
     try:
         import tiktoken  # type: ignore
-        # Use o200k_base encoding for GPT-5 models (fallback to cl100k_base for GPT-4)
-        try:
-            enc = tiktoken.get_encoding("o200k_base")
-        except:
-            enc = tiktoken.get_encoding("cl100k_base")
+        enc = tiktoken.encoding_for_model(model)
         toks = enc.encode(s)
         if len(toks) <= max_tokens:
             return s