fix: Add debug logging and follow-up question support

Debugging improvements:
1. Added debug_grant() function to log complete grant data
2. Debug logging for first grant in search results
3. Debug logging for specific grant details

Follow-up support:
1. smart_route() now accepts previous_grants parameter
2. Detects short follow-up questions (≤8 words with keywords like "sure", "more", "details")
3. Reuses previous grants for context
4. generate_response() tracks last grants on function object

Diagnostic tools:
- debug_data.py script to verify data loading
- Checks snapshots, search index, and diagnoses issues
- Helps identify if problem is in data, index, or context building

Expected improvements:
- Follow-ups like "you sure?" now reference previous grants
- Console logs show what data is actually loaded
- Can quickly diagnose missing funding data

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

debug_data.py

@@ -0,0 +1,102 @@
+"""
+Debug script to verify grant data is loading correctly.
+Run with: python debug_data.py
+"""
+
+from pathlib import Path
+from src.analyzer.data_loader import load_current_grants
+from src.analyzer.search.service import get_grant_by_id
+
+print("=" * 60)
+print("GRANT DATA DEBUG")
+print("=" * 60)
+
+# Load all grants
+print("\n1. Loading grants from snapshots...")
+grants = load_current_grants(Path('data/snapshots'))
+print(f"   ✓ Loaded {len(grants)} grants")
+
+# Check grant 2313 specifically
+print("\n2. Checking competition-2313 (Battery Feasibility)...")
+grant_2313 = next((g for g in grants if '2313' in g.id), None)
+
+if grant_2313:
+    print(f"   ✓ Found grant: {grant_2313.title}")
+    print(f"   Status: {grant_2313.status}")
+    print(f"   Has funding: {grant_2313.funding is not None}")
+
+    if grant_2313.funding:
+        print(f"   Funding min: £{grant_2313.funding.min:,.0f}" if grant_2313.funding.min else "   Funding min: None")
+        print(f"   Funding max: £{grant_2313.funding.max:,.0f}" if grant_2313.funding.max else "   Funding max: None")
+        print(f"   Total pot: £{grant_2313.funding.total_pot:,.0f}" if grant_2313.funding.total_pot else "   Total pot: None")
+    else:
+        print("   ❌ NO FUNDING DATA")
+
+    print(f"   Has summary: {grant_2313.summary is not None and len(grant_2313.summary) > 0}")
+    print(f"   Close date: {grant_2313.close_date}")
+    print(f"   URL: {grant_2313.url}")
+else:
+    print("   ❌ Grant 2313 not found in loaded grants!")
+
+# Check grant 2314
+print("\n3. Checking competition-2314 (Battery Concept)...")
+grant_2314 = next((g for g in grants if '2314' in g.id), None)
+
+if grant_2314:
+    print(f"   ✓ Found grant: {grant_2314.title}")
+    print(f"   Status: {grant_2314.status}")
+
+    if grant_2314.funding:
+        print(f"   Funding min: £{grant_2314.funding.min:,.0f}" if grant_2314.funding.min else "   Funding min: None")
+        print(f"   Funding max: £{grant_2314.funding.max:,.0f}" if grant_2314.funding.max else "   Funding max: None")
+    else:
+        print("   ❌ NO FUNDING DATA")
+else:
+    print("   ❌ Grant 2314 not found!")
+
+# Check search index
+print("\n4. Checking search index (get_grant_by_id)...")
+try:
+    indexed_grant = get_grant_by_id("competition-2313")
+    if indexed_grant:
+        print(f"   ✓ Found via search index: {indexed_grant.title}")
+        print(f"   Status via index: {indexed_grant.status}")
+        print(f"   Has funding via index: {indexed_grant.funding is not None}")
+
+        if indexed_grant.funding:
+            print(f"   Funding via index: £{indexed_grant.funding.min:,.0f} - £{indexed_grant.funding.max:,.0f}")
+        else:
+            print("   ❌ NO FUNDING DATA VIA INDEX")
+    else:
+        print("   ❌ Not found via search index")
+except Exception as e:
+    print(f"   ❌ Error accessing index: {e}")
+
+# Summary
+print("\n" + "=" * 60)
+print("DIAGNOSIS:")
+print("=" * 60)
+
+has_data = grant_2313 and grant_2313.funding and grant_2313.funding.min
+has_indexed_data = False
+
+try:
+    indexed_grant = get_grant_by_id("competition-2313")
+    has_indexed_data = indexed_grant and indexed_grant.funding and indexed_grant.funding.min
+except:
+    pass
+
+if has_data and has_indexed_data:
+    print("✓ Data is present and index is working")
+    print("  Issue is likely in context building or LLM prompt")
+elif has_data and not has_indexed_data:
+    print("✓ Data exists in snapshots")
+    print("❌ Search index is not preserving the data")
+    print("  ACTION: Rebuild search index")
+elif not has_data:
+    print("❌ Data missing from snapshot files")
+    print("  ACTION: Re-run scraper to get funding data")
+else:
+    print("? Unclear - check logs above")
+
+print("=" * 60)

src/analyzer/chat/demo_app.py

@@ -28,6 +28,36 @@ from pathlib import Path
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+
+def debug_grant(grant: Grant) -> str:
+    """Debug helper: Show what data is actually in a grant object"""
+    debug_lines = [
+        f"Grant Debug Info:",
+        f"  ID: {grant.id}",
+        f"  Title: {grant.title}",
+        f"  Status: {grant.status}",
+        f"  Has funding object: {grant.funding is not None}",
+    ]
+
+    if grant.funding:
+        debug_lines.extend([
+            f"  Funding.min: {grant.funding.min}",
+            f"  Funding.max: {grant.funding.max}",
+            f"  Funding.total_pot: {grant.funding.total_pot}",
+        ])
+    else:
+        debug_lines.append("  No funding data")
+
+    debug_lines.extend([
+        f"  Has summary: {grant.summary is not None and len(grant.summary) > 0}",
+        f"  Has eligibility: {grant.eligibility is not None and len(grant.eligibility) > 0}",
+        f"  Close date: {grant.close_date}",
+        f"  URL: {grant.url}",
+    ])
+
+    return "\n".join(debug_lines)
+
+
 # Global state
 llm_client = LLMClient()
 ALL_GRANTS = None
@@ -62,10 +92,13 @@ def cached_search(query: str, limit: int = 10) -> tuple:
         return tuple()
 
 
-def smart_route(query: str) -> Tuple[str, List[Grant], str]:
+def smart_route(query: str, previous_grants: Optional[List[Grant]] = None) -> Tuple[str, List[Grant], str]:
     """
-    Route query to appropriate handler using pattern matching.
-    No LLM overhead - pure pattern matching for speed.
+    Route query with support for follow-up questions.
+
+    Args:
+        query: User query
+        previous_grants: Grants from previous query (for follow-ups)
 
     Returns: (route_type, relevant_grants, debug_note)
 
@@ -80,11 +113,26 @@ def smart_route(query: str) -> Tuple[str, List[Grant], str]:
     # Extract any competition IDs mentioned
     comp_ids = re.findall(r'(?:competition-)?(\d+)', q)
 
+    # NEW: Route 0 - Vague follow-up questions
+    # If query is very short and we have previous context, reuse it
+    if previous_grants and len(q.split()) <= 8:  # Short query
+        follow_up_keywords = [
+            'sure', 'certain', 'really', 'positive',
+            'more', 'extra', 'additional', 'else', 'other',
+            'details', 'info', 'information',
+            'about them', 'about those', 'about that',
+            'tell me more', 'what about', 'how about'
+        ]
+        if any(keyword in q for keyword in follow_up_keywords):
+            logger.info(f"Follow-up detected, reusing {len(previous_grants)} previous grants")
+            return ('detail', previous_grants, f"Follow-up about previous {len(previous_grants)} grants")
+
     # Route 1: Specific grant detail
     if comp_ids and any(word in q for word in ['about', 'detail', 'tell me', 'explain', 'describe']):
         grant_id = f"competition-{comp_ids[0]}"
         grant = get_grant_by_id(grant_id)
         if grant:
+            logger.info(f"\n{debug_grant(grant)}")  # Debug logging
             return ('detail', [grant], f"Detail view for {grant_id}")
         else:
             return ('search', [], f"Grant {grant_id} not found, falling back to search")
@@ -116,6 +164,8 @@ def smart_route(query: str) -> Tuple[str, List[Grant], str]:
         for grant_id, score in cached_results:
             g = get_grant_by_id(grant_id)
             if g:
+                if len(grants) == 0:  # Only debug first grant to avoid spam
+                    logger.info(f"\n{debug_grant(g)}")
                 grants.append(g)
 
         return ('search', grants, f"Found {len(grants)} grants via semantic search")
@@ -258,10 +308,16 @@ def generate_response(query: str, history: List[Tuple[str, str]]) -> str:
     start_time = time.time()
 
     try:
-        # Step 1: Fast routing (no LLM)
-        route_type, grants, debug_note = smart_route(query)
+        # Extract previous grants from conversation state if available
+        previous_grants = getattr(generate_response, '_last_grants', None)
+
+        # Step 1: Fast routing with follow-up support
+        route_type, grants, debug_note = smart_route(query, previous_grants)
         logger.info(f"Route: {route_type} | Grants: {len(grants)} | Note: {debug_note}")
 
+        # Store grants for next follow-up
+        generate_response._last_grants = grants
+
         # Step 2: Handle no results
         if not grants:
             return "I couldn't find any grants matching your query. Try different keywords, or ask about a specific competition ID (e.g., 'tell me about competition-2313')."