Initial commit: UA Student Navigator Chatbot with OpenRouter integration

.gitignore

@@ -0,0 +1,33 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+venv/
+env/
+ENV/
+.venv
+
+# Environment variables
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+searxng-data/
+*.log
+
+# Hugging Face
+.huggingface/
+

README_HF.md

@@ -0,0 +1,67 @@
+---
+title: UA Student Navigator Chatbot
+emoji: 🎓
+colorFrom: blue
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.28.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+# 🎓 UA Student Navigator Chatbot
+
+A web-retrieval + summarization agent that answers University of Arizona (UA) student questions using only official UA webpages and always shows sources.
+
+## Features
+
+- 🔍 **SearXNG Integration**: Searches UA domains (arizona.edu and subdomains)
+- 📄 **Full Page Reading**: Fetches and extracts complete content from webpages
+- 🤖 **OpenRouter Integration**: Access to 30+ free, open-source models
+- 🕰️ **Wayback Machine**: Automatic fallback for blocked pages
+- 🕷️ **Controlled Crawling**: Optionally crawls internal links from user-provided URLs
+- 📊 **Evidence-Based**: All answers are grounded in extracted evidence with citations
+- 🚫 **No Hallucinations**: Explicitly states when information is not found
+- ⚡ **Rate Limit Management**: Smart rate limiting with partial results
+
+## Getting Started
+
+1. **Get an OpenRouter API Key** (free at https://openrouter.ai)
+2. **Enter your API key** in the sidebar
+3. **Select a model** from the dropdown (30+ free models available)
+4. **Ask questions** about University of Arizona!
+
+## Usage
+
+### Search Query
+Ask questions like:
+- "What campus resources are available?"
+- "What are the housing deadlines?"
+- "What majors are in the College of Science?"
+
+### Website URL
+Analyze specific UA webpages by providing a URL and what you're looking for.
+
+## Available Models
+
+The app automatically loads free, open-source models including:
+- `xiaomi/mimo-v2-flash:free` - 262K context
+- `deepseek/deepseek-r1-0528:free` - 163K context, reasoning
+- `openai/gpt-oss-20b:free` - 131K context
+- And many more!
+
+## Rate Limits
+
+The app automatically manages rate limits and will show partial results if limits are reached. Try different models if one is rate-limited.
+
+## Privacy
+
+- All API keys are stored locally in your session
+- No data is stored or logged
+- All searches are limited to arizona.edu domains
+
+## License
+
+MIT License - See LICENSE file for details
+

app.py

@@ -0,0 +1,407 @@
+"""
+UA Student Navigator Chatbot - Main Streamlit App
+Now with OpenRouter integration for free open-source models!
+"""
+import streamlit as st
+import os
+import time
+from dotenv import load_dotenv
+from typing import List, Dict, Optional
+
+# Load environment variables
+load_dotenv()
+
+# Import our modules
+from search import ua_search
+from fetch import fetch_page
+from extract import extract_evidence
+from rank import rank_pages, rank_snippets
+from llm import get_openrouter_models, generate_answer_openrouter, check_ollama_available, check_model_exists
+from crawl import crawl_website
+from rate_limit import get_rate_limit_tracker
+
+
+# Page configuration
+st.set_page_config(
+    page_title="UA Student Navigator",
+    page_icon="🎓",
+    layout="wide"
+)
+
+# Initialize session state
+if 'search_results' not in st.session_state:
+    st.session_state.search_results = []
+if 'evidence' not in st.session_state:
+    st.session_state.evidence = None
+if 'available_models' not in st.session_state:
+    st.session_state.available_models = []
+
+
+def get_available_models(api_key: str):
+    """Get and cache available OpenRouter models."""
+    if not st.session_state.available_models:
+        with st.spinner("Loading available models..."):
+            models = get_openrouter_models(api_key)
+            st.session_state.available_models = models
+    return st.session_state.available_models
+
+
+def process_search_query(query: str, max_results: int = 8, model: Optional[str] = None, api_key: Optional[str] = None):
+    """Process a search query and return evidence."""
+    # Adjust max_results based on rate limits if model is provided
+    if model and api_key:
+        rate_tracker = get_rate_limit_tracker()
+        max_searches = rate_tracker.calculate_max_searches(model, api_key)
+        if max_searches < max_results:
+            st.warning(f"⚠️ Rate limit: Reducing search results from {max_results} to {max_searches} to avoid rate limits")
+            max_results = max_searches
+    
+    with st.spinner("Searching UA websites..."):
+        search_results = ua_search(query, max_results=max_results)
+        
+        if not search_results:
+            return None, """No search results found. This could be because:
+1. SearXNG is not accessible (check if it's running or try a different instance)
+2. No UA pages matched your query
+
+**You can still use the app:**
+- Try the "🌐 Website URL" tab to analyze a specific UA webpage directly"""
+        
+        st.info(f"Found {len(search_results)} search results. Fetching pages...")
+        
+        pages = []
+        rate_limited = False
+        for i, result in enumerate(search_results):
+            # Check rate limit before each fetch if model is provided
+            if model and api_key:
+                rate_tracker = get_rate_limit_tracker()
+                can_request, rate_limit_msg = rate_tracker.can_make_request(model, api_key)
+                if not can_request:
+                    st.warning(f"⚠️ Rate limit reached after processing {i} pages. Showing partial results.")
+                    rate_limited = True
+                    break
+            
+            with st.spinner(f"Fetching: {result['url']}"):
+                page = fetch_page(result['url'])
+                if page:
+                    pages.append(page)
+        
+        if not pages:
+            return None, "Could not fetch any pages. They may be blocked or unavailable."
+        
+        evidence = extract_evidence(pages, query)
+        ranked_pages = rank_pages(pages, query)
+        evidence['pages'] = ranked_pages
+        
+        ranked_snippets = rank_snippets(evidence['snippets'], query)
+        evidence['snippets'] = ranked_snippets[:25]
+        evidence['rate_limited'] = rate_limited
+        
+        return evidence, None
+
+
+def process_url_query(url: str, query: str, max_pages: int = 12, max_depth: int = 1):
+    """Process a URL query with optional crawling."""
+    with st.spinner(f"Fetching and analyzing: {url}"):
+        page = fetch_page(url)
+        if not page:
+            return None, f"Could not fetch the page at {url}. Please check the URL."
+        
+        pages = [page]
+        
+        if max_pages > 1 and max_depth > 0:
+            st.info(f"Crawling internal links (max {max_pages} pages, depth {max_depth})...")
+            crawled_pages = crawl_website(url, query, max_pages=max_pages, max_depth=max_depth)
+            seen_urls = {page['url']}
+            for p in crawled_pages:
+                if p['url'] not in seen_urls:
+                    pages.append(p)
+                    seen_urls.add(p['url'])
+        
+        evidence = extract_evidence(pages, query)
+        ranked_pages = rank_pages(pages, query)
+        evidence['pages'] = ranked_pages
+        
+        ranked_snippets = rank_snippets(evidence['snippets'], query)
+        evidence['snippets'] = ranked_snippets[:25]
+        
+        return evidence, None
+
+
+def display_results(evidence: Dict, query: str, model: str, api_key: str, use_openrouter: bool = True):
+    """Display results in the UI."""
+    if not evidence:
+        return
+    
+    # Generate answer
+    snippets = evidence.get('snippets', [])
+    source_urls = list(set([s.get('source_url', '') for s in snippets]))
+    
+    if snippets:
+        st.info(f"📊 Using {len(snippets)} evidence snippets from {len(source_urls)} sources to generate answer...")
+    
+    # Show progress for long-running requests
+    progress_bar = st.progress(0)
+    status_text = st.empty()
+    
+    try:
+        status_text.text("🔄 Connecting to model...")
+        progress_bar.progress(10)
+        
+        if use_openrouter:
+            status_text.text("🤖 Generating answer with OpenRouter...")
+            progress_bar.progress(30)
+            partial_evidence = evidence if evidence.get('rate_limited') else None
+            answer, was_rate_limited, rate_limit_msg = generate_answer_openrouter(
+                query, snippets, source_urls, model, api_key, partial_evidence
+            )
+            
+            if was_rate_limited:
+                st.warning(f"⚠️ **Rate Limit Reached**\n\n{rate_limit_msg or 'Maximum requests per minute reached for this model.'}")
+                st.info("💡 **Tip:** Try a different model or wait a moment before trying again.")
+        else:
+            status_text.text("🖥️ Generating answer with Ollama...")
+            progress_bar.progress(30)
+            from llm import generate_answer
+            ollama_host = os.getenv('OLLAMA_HOST', 'http://localhost:11434')
+            answer = generate_answer(query, snippets, source_urls, ollama_host, model)
+            was_rate_limited = False
+            rate_limit_msg = None
+        
+        progress_bar.progress(100)
+        status_text.text("✅ Answer generated!")
+        
+    except Exception as e:
+        progress_bar.progress(0)
+        status_text.text(f"❌ Error: {str(e)}")
+        answer = f"Error generating answer: {str(e)}"
+        was_rate_limited = False
+        rate_limit_msg = None
+    
+    finally:
+        # Clear progress after a moment
+        time.sleep(0.5)
+        progress_bar.empty()
+        status_text.empty()
+    
+    # Display Final Answer
+    st.header("📝 Final Answer")
+    st.markdown(answer)
+    
+    # Key Findings
+    st.header("🔍 Key Findings")
+    findings = evidence.get('snippets', [])[:10]
+    if findings:
+        for i, finding in enumerate(findings, 1):
+            st.markdown(f"**{i}.** {finding.get('text', '')}")
+            st.caption(f"Source: [{finding.get('source_title', 'Link')}]({finding.get('source_url', '')})")
+    else:
+        st.info("No specific findings extracted.")
+    
+    # Dates Found
+    dates = evidence.get('dates', [])
+    if dates:
+        st.header("📅 Dates Found")
+        for date_info in dates[:10]:
+            st.markdown(f"**{date_info.get('date', 'Unknown')}**")
+            st.markdown(f"*Context:* {date_info.get('context', '')}")
+            st.caption(f"Source: [{date_info.get('source_title', 'Link')}]({date_info.get('source_url', '')})")
+    
+    # Sources
+    st.header("📚 Sources")
+    pages = evidence.get('pages', [])
+    if pages:
+        for i, page in enumerate(pages[:10], 1):
+            score = page.get('relevance_score', 0)
+            source_indicator = ""
+            if page.get('source') == 'wayback_machine':
+                source_indicator = " (via Wayback Machine)"
+            st.markdown(f"**{i}.** [{page.get('title', 'No title')}]({page.get('url', '')}){source_indicator}")
+            st.caption(f"Relevance score: {score:.2f}")
+
+
+# Main UI
+st.title("🎓 UA Student Navigator Chatbot")
+st.markdown("Ask questions about University of Arizona using official UA webpages only.")
+
+# Sidebar for configuration
+with st.sidebar:
+    st.header("⚙️ Configuration")
+    
+    # OpenRouter API Key
+    default_key = os.getenv('OPENROUTER_API_KEY', '')
+    api_key = st.text_input(
+        "OpenRouter API Key",
+        value=default_key,
+        type="password",
+        help="Your OpenRouter API key"
+    )
+    
+    if not api_key or api_key.strip() == '':
+        st.warning("⚠️ Please enter your OpenRouter API key")
+        st.stop()
+    
+    api_key = api_key.strip()  # Remove any whitespace
+    
+    # Model Selection
+    st.subheader("🤖 Model Selection")
+    use_openrouter = st.radio(
+        "LLM Provider",
+        ["OpenRouter (Free Open-Source Models)", "Ollama (Local)"],
+        index=0,
+        help="Choose between OpenRouter cloud models or local Ollama"
+    )
+    
+    if use_openrouter == "OpenRouter (Free Open-Source Models)":
+        # Get available models
+        models = get_available_models(api_key)
+        
+        if models:
+            model_options = {f"{m['name']} ({m['context_length']:,} tokens)": m['id'] for m in models[:15]}
+            selected_model_name = st.selectbox(
+                "Select Model",
+                options=list(model_options.keys()),
+                index=0,
+                help="Free open-source models available via OpenRouter"
+            )
+            selected_model = model_options[selected_model_name]
+            
+            # Show model info and rate limits
+            selected_model_info = next((m for m in models if m['id'] == selected_model), None)
+            if selected_model_info:
+                rate_tracker = get_rate_limit_tracker()
+                model_limits = rate_tracker.get_model_rate_limit(selected_model, api_key)
+                remaining = rate_tracker.get_remaining_requests(selected_model, api_key)
+                max_searches = rate_tracker.calculate_max_searches(selected_model, api_key)
+                
+                with st.expander("Model Details"):
+                    st.write(f"**ID:** {selected_model_info['id']}")
+                    st.write(f"**Context Length:** {selected_model_info['context_length']:,} tokens")
+                    st.write(f"**Description:** {selected_model_info['description']}")
+                    st.write("---")
+                    st.write("**Rate Limits:**")
+                    st.write(f"- Requests per minute: {model_limits.get('requests_per_minute', 'N/A')}")
+                    st.write(f"- Remaining requests: {remaining}/{model_limits.get('requests_per_minute', 'N/A')}")
+                    st.write(f"- Max searches recommended: {max_searches}")
+        else:
+            st.error("Could not load models. Check your API key.")
+            st.stop()
+    else:
+        # Ollama configuration
+        ollama_host = st.text_input(
+            "Ollama Host",
+            value=os.getenv('OLLAMA_HOST', 'http://localhost:11434'),
+            help="Ollama API endpoint"
+        )
+        os.environ['OLLAMA_HOST'] = ollama_host
+        
+        ollama_model = st.text_input(
+            "Ollama Model",
+            value=os.getenv('OLLAMA_MODEL', 'qwen2.5:7b'),
+            help="Model name (must be pulled locally)"
+        )
+        os.environ['OLLAMA_MODEL'] = ollama_model
+        
+        selected_model = ollama_model
+        
+        # Check Ollama status
+        available, error = check_ollama_available(ollama_host)
+        if available:
+            st.success("✓ Ollama is running")
+            exists, error, suggested = check_model_exists(ollama_model, ollama_host)
+            if exists:
+                st.success(f"✓ Model '{ollama_model}' is available")
+            else:
+                st.error(f"✗ {error}")
+        else:
+            st.error("✗ Ollama not running")
+    
+    # SearXNG Configuration
+    st.subheader("🔍 SearXNG Settings")
+    searxng_url = st.text_input(
+        "SearXNG URL",
+        value=os.getenv('SEARXNG_URL', 'http://localhost:8080'),
+        help="URL of your SearXNG instance"
+    )
+    os.environ['SEARXNG_URL'] = searxng_url
+    
+    st.markdown("---")
+    if st.button("🔄 Refresh Models"):
+        st.session_state.available_models = []
+        st.rerun()
+
+
+# Main interface
+tab1, tab2 = st.tabs(["🔍 Search Query", "🌐 Website URL"])
+
+with tab1:
+    st.header("Ask a Question")
+    query = st.text_input(
+        "Enter your question about UA:",
+        placeholder="e.g., What are the deadlines for housing?",
+        key="search_query"
+    )
+    
+    max_results = st.slider("Max search results", 3, 15, 8)
+    
+    if st.button("Search", type="primary"):
+        if not query:
+            st.warning("Please enter a question.")
+        else:
+            # Pass model and API key for rate limit management
+            selected_model_for_search = selected_model if use_openrouter == "OpenRouter (Free Open-Source Models)" else None
+            evidence, error = process_search_query(query, max_results=max_results, model=selected_model_for_search, api_key=api_key)
+            
+            if error:
+                st.error(error)
+            elif evidence:
+                st.session_state.evidence = evidence
+                use_openrouter_flag = use_openrouter == "OpenRouter (Free Open-Source Models)"
+                display_results(evidence, query, selected_model, api_key, use_openrouter_flag)
+            else:
+                st.error("No evidence extracted.")
+
+with tab2:
+    st.header("Analyze a UA Webpage")
+    url = st.text_input(
+        "Enter a UA webpage URL:",
+        placeholder="https://global.arizona.edu/...",
+        key="url_input"
+    )
+    
+    url_query = st.text_input(
+        "What information are you looking for?",
+        placeholder="e.g., What do international students need to do for CPT?",
+        key="url_query"
+    )
+    
+    col1, col2 = st.columns(2)
+    with col1:
+        max_pages = st.number_input("Max pages to crawl", 1, 20, 12)
+    with col2:
+        max_depth = st.number_input("Max crawl depth", 0, 2, 1)
+    
+    if st.button("Analyze Website", type="primary"):
+        if not url:
+            st.warning("Please enter a URL.")
+        elif not url_query:
+            st.warning("Please enter what you're looking for.")
+        else:
+            evidence, error = process_url_query(url, url_query, max_pages=max_pages, max_depth=max_depth)
+            
+            if error:
+                st.error(error)
+            elif evidence:
+                st.session_state.evidence = evidence
+                use_openrouter_flag = use_openrouter == "OpenRouter (Free Open-Source Models)"
+                display_results(evidence, url_query, selected_model, api_key, use_openrouter_flag)
+            else:
+                st.error("No evidence extracted.")
+
+# Display cached results if available
+if st.session_state.evidence:
+    st.sidebar.markdown("---")
+    if st.sidebar.button("Clear Results"):
+        st.session_state.evidence = None
+        st.rerun()
+

crawl.py

@@ -0,0 +1,55 @@
+"""
+Controlled website crawler for user-provided URLs.
+"""
+from typing import List, Dict, Set
+from urllib.parse import urlparse, urljoin
+from fetch import fetch_page, get_internal_links
+import time
+
+
+def crawl_website(start_url: str, query: str, max_pages: int = 12, max_depth: int = 1) -> List[Dict[str, str]]:
+    """Crawl a website starting from a URL, following internal links."""
+    parsed_start = urlparse(start_url)
+    base_domain = parsed_start.netloc.lower()
+    base_path = parsed_start.path.rstrip('/')
+    
+    pages = []
+    visited: Set[str] = set()
+    to_visit: List[tuple[str, int]] = [(start_url, 0)]
+    
+    while to_visit and len(pages) < max_pages:
+        current_url, depth = to_visit.pop(0)
+        
+        if current_url in visited:
+            continue
+        
+        if depth > max_depth:
+            continue
+        
+        print(f"Fetching (depth {depth}): {current_url}")
+        page = fetch_page(current_url)
+        
+        if page:
+            pages.append(page)
+            visited.add(current_url)
+            
+            if depth < max_depth and len(pages) < max_pages:
+                try:
+                    import httpx
+                    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+                    with httpx.Client(timeout=10.0, follow_redirects=True) as client:
+                        response = client.get(current_url, headers=headers)
+                        if response.status_code == 200 and 'text/html' in response.headers.get('content-type', '').lower():
+                            html = response.text
+                            links = get_internal_links(html, current_url, same_domain_only=True)
+                            
+                            for link in links:
+                                if link not in visited:
+                                    to_visit.append((link, depth + 1))
+                except Exception as e:
+                    print(f"Error getting links from {current_url}: {e}")
+        
+        time.sleep(0.5)
+    
+    return pages
+

extract.py

@@ -0,0 +1,124 @@
+"""
+Extract evidence, dates, and relevant snippets from content.
+"""
+import re
+from typing import List, Dict
+from datetime import datetime
+
+
+def extract_dates(text: str) -> List[Dict[str, str]]:
+    """Extract dates from text in various formats."""
+    dates = []
+    
+    pattern1 = r'\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),?\s+(\d{4})\b'
+    for match in re.finditer(pattern1, text, re.IGNORECASE):
+        month, day, year = match.groups()
+        date_str = f"{month} {day}, {year}"
+        start = max(0, match.start() - 50)
+        end = min(len(text), match.end() + 50)
+        context = text[start:end].strip()
+        dates.append({'date': date_str, 'context': context, 'raw': match.group()})
+    
+    pattern2 = r'\b(\d{1,2})[/-](\d{1,2})[/-](\d{4})\b'
+    for match in re.finditer(pattern2, text):
+        month, day, year = match.groups()
+        date_str = f"{month}/{day}/{year}"
+        start = max(0, match.start() - 50)
+        end = min(len(text), match.end() + 50)
+        context = text[start:end].strip()
+        dates.append({'date': date_str, 'context': context, 'raw': match.group()})
+    
+    pattern3 = r'\b(\d{4})-(\d{1,2})-(\d{1,2})\b'
+    for match in re.finditer(pattern3, text):
+        year, month, day = match.groups()
+        date_str = f"{month}/{day}/{year}"
+        start = max(0, match.start() - 50)
+        end = min(len(text), match.end() + 50)
+        context = text[start:end].strip()
+        dates.append({'date': date_str, 'context': context, 'raw': match.group()})
+    
+    pattern4 = r'\b(Spring|Summer|Fall|Winter)\s+(\d{4})\b'
+    for match in re.finditer(pattern4, text, re.IGNORECASE):
+        season, year = match.groups()
+        date_str = f"{season} {year}"
+        start = max(0, match.start() - 50)
+        end = min(len(text), match.end() + 50)
+        context = text[start:end].strip()
+        dates.append({'date': date_str, 'context': context, 'raw': match.group()})
+    
+    seen = set()
+    unique_dates = []
+    for d in dates:
+        if d['date'] not in seen:
+            seen.add(d['date'])
+            unique_dates.append(d)
+    
+    return unique_dates
+
+
+def extract_relevant_snippets(content: str, query: str, max_snippets: int = 15, snippet_length: int = 300) -> List[str]:
+    """Extract relevant snippets from content based on query keywords."""
+    keywords = [w.lower() for w in query.split() if len(w) > 3]
+    
+    if not keywords:
+        sentences = re.split(r'[.!?]+', content)
+        return [s.strip()[:snippet_length] for s in sentences[:max_snippets] if s.strip()]
+    
+    sentences = re.split(r'[.!?]+', content)
+    scored_sentences = []
+    for sentence in sentences:
+        sentence_lower = sentence.lower()
+        score = sum(1 for keyword in keywords if keyword in sentence_lower)
+        if score > 0:
+            scored_sentences.append((score, sentence.strip()))
+    
+    scored_sentences.sort(reverse=True, key=lambda x: x[0])
+    snippets = []
+    seen = set()
+    
+    for score, sentence in scored_sentences[:max_snippets * 2]:
+        snippet = sentence[:snippet_length]
+        if snippet not in seen and len(snippet) > 50:
+            seen.add(snippet)
+            snippets.append(snippet)
+            if len(snippets) >= max_snippets:
+                break
+    
+    if len(snippets) < max_snippets:
+        for sentence in sentences:
+            snippet = sentence.strip()[:snippet_length]
+            if snippet not in seen and len(snippet) > 50:
+                seen.add(snippet)
+                snippets.append(snippet)
+                if len(snippets) >= max_snippets:
+                    break
+    
+    return snippets
+
+
+def extract_evidence(pages: List[Dict[str, str]], query: str) -> Dict:
+    """Extract evidence from multiple pages."""
+    all_snippets = []
+    all_dates = []
+    
+    for page in pages:
+        content = page.get('content', '')
+        if not content:
+            continue
+        
+        snippets = extract_relevant_snippets(content, query, max_snippets=8, snippet_length=300)
+        for snippet in snippets:
+            all_snippets.append({
+                'text': snippet,
+                'source_url': page['url'],
+                'source_title': page.get('title', 'No title')
+            })
+        
+        dates = extract_dates(content)
+        for date_info in dates:
+            date_info['source_url'] = page['url']
+            date_info['source_title'] = page.get('title', 'No title')
+            all_dates.append(date_info)
+    
+    return {'snippets': all_snippets, 'dates': all_dates, 'pages': pages}
+

fetch.py

@@ -0,0 +1,267 @@
+"""
+Fetch and extract content from web pages.
+Includes Wayback Machine fallback for 403 errors.
+"""
+import httpx
+from bs4 import BeautifulSoup
+from typing import Optional, Dict
+from urllib.parse import urljoin, urlparse
+import re
+import json
+import time
+
+
+def clean_text(text: str) -> str:
+    """Clean extracted text."""
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+def extract_main_content(html: str, url: str, max_chars: int = 120000) -> str:
+    """Extract main content from HTML, removing scripts, styles, nav, footer."""
+    soup = BeautifulSoup(html, 'lxml')
+    
+    for script in soup(['script', 'style', 'noscript']):
+        script.decompose()
+    
+    for nav in soup.find_all(['nav', 'header', 'footer', 'aside']):
+        nav.decompose()
+    
+    junk_patterns = ['nav', 'navigation', 'menu', 'sidebar', 'footer', 'header', 'cookie', 'banner', 'advertisement', 'ad-', 'social-']
+    
+    for element in soup.find_all(class_=re.compile('|'.join(junk_patterns), re.I)):
+        element.decompose()
+    
+    for element in soup.find_all(id=re.compile('|'.join(junk_patterns), re.I)):
+        element.decompose()
+    
+    main_content = None
+    main_selectors = ['main', 'article', '[role="main"]', '.content', '.main-content', '.post-content', '#content', '#main-content', '#main']
+    
+    for selector in main_selectors:
+        main_content = soup.select_one(selector)
+        if main_content:
+            break
+    
+    if not main_content:
+        main_content = soup.find('body')
+    
+    if not main_content:
+        return ""
+    
+    text = main_content.get_text(separator=' ', strip=True)
+    text = clean_text(text)
+    
+    if len(text) > max_chars:
+        text = text[:max_chars] + "... [truncated]"
+    
+    return text
+
+
+def get_wayback_snapshot(url: str, timeout: float = 20.0) -> Optional[str]:
+    """Get the latest Wayback Machine snapshot URL for a given URL."""
+    try:
+        api_url = f"https://web.archive.org/cdx/search/cdx?url={url}&output=json&limit=1&collapse=urlkey"
+        headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
+        
+        for attempt in range(2):
+            try:
+                with httpx.Client(timeout=timeout, headers=headers) as client:
+                    response = client.get(api_url)
+                    response.raise_for_status()
+                    data = response.json()
+                    if len(data) > 1:
+                        timestamp = data[1][1]
+                        wayback_url = f"https://web.archive.org/web/{timestamp}/{url}"
+                        return wayback_url
+                    break
+            except httpx.TimeoutException:
+                if attempt == 0:
+                    print(f"Wayback API timeout, retrying...")
+                    time.sleep(1)
+                    continue
+                else:
+                    raise
+            except Exception as e:
+                if attempt == 0:
+                    print(f"Wayback API error, retrying...")
+                    time.sleep(1)
+                    continue
+                else:
+                    raise
+        
+        return None
+    except Exception as e:
+        print(f"Error getting Wayback snapshot for {url}: {e}")
+        return None
+
+
+def extract_wayback_content(html: str) -> Optional[str]:
+    """Extract original content from Wayback Machine wrapper HTML."""
+    try:
+        soup = BeautifulSoup(html, 'html.parser')
+        content_divs = soup.find_all('div', id='webpage')
+        if not content_divs:
+            content_divs = soup.find_all('div', class_='webpage')
+        if not content_divs:
+            content_divs = soup.find_all('div', {'id': re.compile('content|main', re.I)})
+        
+        if content_divs:
+            return str(content_divs[0])
+        
+        body = soup.find('body')
+        if body:
+            for elem in body.find_all(['div', 'script', 'style'], class_=re.compile('wm-|wayback', re.I)):
+                elem.decompose()
+            return str(body)
+        
+        return html
+    except Exception as e:
+        print(f"Error extracting Wayback content: {e}")
+        return html
+
+
+def fetch_page(url: str, timeout: float = 30.0, use_wayback_fallback: bool = True) -> Optional[Dict[str, str]]:
+    """Fetch a webpage and extract its content. Falls back to Wayback Machine if 403 error occurs."""
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Cache-Control': 'max-age=0'
+    }
+    
+    try:
+        with httpx.Client(timeout=timeout, follow_redirects=True, headers=headers) as client:
+            response = client.get(url, headers=headers)
+            response.raise_for_status()
+            
+            content_type = response.headers.get('content-type', '').lower()
+            if 'text/html' not in content_type:
+                print(f"Skipping non-HTML content: {content_type}")
+                return None
+            
+            html = response.text
+            soup = BeautifulSoup(html, 'lxml')
+            title_tag = soup.find('title')
+            title = title_tag.get_text(strip=True) if title_tag else 'No title'
+            content = extract_main_content(html, url)
+            
+            if not content or len(content) < 100:
+                print(f"Warning: Very little content extracted from {url}")
+            
+            return {'url': url, 'title': title, 'content': content}
+    
+    except httpx.TimeoutException:
+        print(f"Timeout fetching {url}")
+        return None
+    except httpx.HTTPStatusError as e:
+        if e.response.status_code == 403 and use_wayback_fallback:
+            print(f"HTTP 403 error fetching {url}, trying Wayback Machine...")
+            return fetch_from_wayback(url, timeout)
+        else:
+            print(f"HTTP error {e.response.status_code} fetching {url}")
+            return None
+    except Exception as e:
+        print(f"Error fetching {url}: {e}")
+        return None
+
+
+def fetch_from_wayback(url: str, timeout: float = 30.0) -> Optional[Dict[str, str]]:
+    """Fetch a webpage from Wayback Machine (Internet Archive)."""
+    try:
+        wayback_url = get_wayback_snapshot(url, timeout=10.0)
+        if not wayback_url:
+            print(f"No Wayback Machine snapshot found for {url}")
+            return None
+        
+        print(f"Fetching from Wayback Machine: {wayback_url}")
+        headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'}
+        
+        with httpx.Client(timeout=timeout, follow_redirects=True, headers=headers) as client:
+            response = client.get(wayback_url)
+            response.raise_for_status()
+            html = response.text
+            original_html = extract_wayback_content(html)
+            if original_html:
+                html = original_html
+            
+            soup = BeautifulSoup(html, 'lxml')
+            title_tag = soup.find('title')
+            title = title_tag.get_text(strip=True) if title_tag else ''
+            
+            title = re.sub(r'^.*?Wayback Machine\s*[:\-]\s*', '', title, flags=re.I)
+            title = re.sub(r'\s*[:\-]\s*Wayback Machine.*?$', '', title, flags=re.I)
+            title = re.sub(r'^.*?Internet Archive\s*[:\-]\s*', '', title, flags=re.I)
+            
+            if not title or title.lower() in ['wayback machine', 'internet archive']:
+                h1 = soup.find('h1')
+                if h1:
+                    title = h1.get_text(strip=True)
+                else:
+                    meta_title = soup.find('meta', property='og:title')
+                    if meta_title:
+                        title = meta_title.get('content', '')
+            
+            if not title or title.lower() in ['wayback machine', 'internet archive', 'no title']:
+                parsed = urlparse(url)
+                title = parsed.path.strip('/').replace('/', ' - ').replace('-', ' ').title()
+                if not title:
+                    title = parsed.netloc.replace('.', ' ').title()
+            
+            if not title:
+                title = 'No title'
+            
+            content = extract_main_content(html, url)
+            
+            if not content or len(content) < 100:
+                print(f"Warning: Very little content extracted from Wayback snapshot for {url}")
+            
+            return {'url': url, 'title': title, 'content': content, 'source': 'wayback_machine'}
+    
+    except httpx.TimeoutException:
+        print(f"Timeout fetching from Wayback Machine: {url}")
+        return None
+    except httpx.HTTPStatusError as e:
+        print(f"HTTP error {e.response.status_code} fetching from Wayback Machine: {url}")
+        return None
+    except Exception as e:
+        print(f"Error fetching from Wayback Machine {url}: {e}")
+        return None
+
+
+def get_internal_links(html: str, base_url: str, same_domain_only: bool = True) -> list:
+    """Extract internal links from HTML."""
+    soup = BeautifulSoup(html, 'lxml')
+    base_parsed = urlparse(base_url)
+    base_domain = base_parsed.netloc.lower()
+    
+    links = []
+    seen = set()
+    
+    for a_tag in soup.find_all('a', href=True):
+        href = a_tag['href']
+        absolute_url = urljoin(base_url, href)
+        parsed = urlparse(absolute_url)
+        clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
+        if parsed.query:
+            clean_url += f"?{parsed.query}"
+        
+        if same_domain_only:
+            if parsed.netloc.lower() != base_domain:
+                continue
+        
+        if clean_url in seen or clean_url == base_url:
+            continue
+        
+        seen.add(clean_url)
+        links.append(clean_url)
+    
+    return links
+

llm.py

@@ -0,0 +1,506 @@
+"""
+LLM integration for answer generation.
+Supports both OpenRouter (free open-source models) and Ollama (local models).
+"""
+import httpx
+import os
+import time
+from typing import List, Dict, Optional, Tuple
+from rate_limit import get_rate_limit_tracker
+
+
+def get_openrouter_models(api_key: str) -> List[Dict[str, str]]:
+    """
+    Get list of free, open-source models from OpenRouter.
+    
+    Returns:
+        List of model dicts with 'id', 'name', 'context_length' keys
+    """
+    try:
+        headers = {
+            'Authorization': f'Bearer {api_key}',
+            'Content-Type': 'application/json'
+        }
+        
+        with httpx.Client(timeout=15.0) as client:
+            response = client.get('https://openrouter.ai/api/v1/models', headers=headers)
+            response.raise_for_status()
+            data = response.json()
+        
+        free_open_source_models = []
+        
+        for model in data.get('data', []):
+            pricing = model.get('pricing', {})
+            prompt_price = pricing.get('prompt', '0')
+            completion_price = pricing.get('completion', '0')
+            
+            # Must be free
+            is_free = (prompt_price == '0' or prompt_price == 0 or prompt_price is None) and \
+                     (completion_price == '0' or completion_price == 0 or completion_price is None)
+            
+            if not is_free:
+                continue
+            
+            model_id = model.get('id', '')
+            name = model.get('name', '')
+            description = model.get('description', '').lower()
+            
+            # Check if open source
+            is_open_source = any(keyword in model_id.lower() or keyword in name.lower() or keyword in description 
+                                for keyword in ['open', 'oss', 'apache', 'mit', 'open-source', 'open source', 
+                                               'openweight', 'open-source', 'mimo', 'deepseek-r1'])
+            
+            # Filter for general-purpose models (not just coding)
+            # But include some good ones even if they mention coding
+            context_len = model.get('context_length', 0)
+            
+            # Include models that are free and (open source OR explicitly marked as free)
+            if is_free and (is_open_source or ':free' in model_id.lower()) and context_len >= 32000:
+                # Skip models that require data policy configuration
+                if 'gpt-oss-120b' in model_id.lower():
+                    continue  # Requires data policy configuration
+                
+                # Prefer general-purpose, but include some coding models that are good
+                if 'coder' in model_id.lower() and 'devstral' not in model_id.lower():
+                    continue  # Skip pure coding models except devstral
+                
+                free_open_source_models.append({
+                    'id': model_id,
+                    'name': name,
+                    'context_length': context_len,
+                    'description': model.get('description', '')[:200]
+                })
+        
+        # Sort by context length (descending)
+        free_open_source_models.sort(key=lambda x: x['context_length'], reverse=True)
+        
+        # Filter out known problematic models
+        reliable_models = []
+        problematic_models = ['gpt-oss-120b:free']  # Requires data policy
+        
+        for model in free_open_source_models:
+            if not any(prob in model['id'].lower() for prob in problematic_models):
+                reliable_models.append(model)
+        
+        return reliable_models
+    
+    except Exception as e:
+        print(f"Error fetching OpenRouter models: {e}")
+        return []
+
+
+def generate_answer_openrouter(
+    query: str,
+    evidence_snippets: List[Dict[str, str]],
+    source_urls: List[str],
+    model: str,
+    api_key: str,
+    partial_evidence: Optional[Dict] = None
+) -> Tuple[str, bool, Optional[str]]:
+    """
+    Generate answer using OpenRouter API with free open-source models.
+    
+    Args:
+        query: User question
+        evidence_snippets: List of dicts with 'text' and 'source_url' keys
+        source_urls: List of source URLs
+        model: Model ID from OpenRouter
+        api_key: OpenRouter API key
+        partial_evidence: Optional partial evidence dict if rate limited
+    
+    Returns:
+        Tuple of (answer_text, was_rate_limited, rate_limit_message)
+    """
+    # Build evidence text - limit to prevent timeouts
+    # Use top 15 snippets to balance context vs speed
+    evidence_text = "\n\n".join([
+        f"[Source: {snippet.get('source_url', 'unknown')}]\n{snippet.get('text', '')}"
+        for snippet in evidence_snippets[:15]  # Limit to 15 for faster responses
+    ])
+    
+    # Count evidence
+    evidence_count = len(evidence_snippets)
+    source_count = len(source_urls)
+    
+    # Build system prompt
+    system_prompt = """You are a professional research assistant for University of Arizona students. Your job is to ensure students obtain comprehensive information about the University of Arizona and everything it has to offer, including but not limited to:
+
+1. Student life
+2. Academic and research opportunities
+3. Clubs and student organizations
+4. Off-campus resources provided by the University of Arizona
+5. Campus resources
+6. Staff, professors, and entertainment spots
+7. Updates on the University of Arizona that affect students
+
+Your mission is to make student life and student research much easier. You have access to research tools (SearXNG) that allow you to search across arizona.edu domains and read through official UA websites.
+
+WORK STRUCTURE:
+• The user asks a question
+• Research has been conducted across arizona.edu domains
+• Websites and URLs have been scraped and read through
+• Key facts and dates matching the user's query have been extracted
+• You now have evidence from official UA sources
+
+YOUR RESPONSIBILITIES:
+1. **USE THE EVIDENCE PROVIDED**: The evidence below comes from official UA webpages. You MUST use this evidence to answer the question.
+2. **BE COMPREHENSIVE**: Extract and present all relevant information from the evidence that answers or relates to the user's question.
+3. **CITE SOURCES**: Every key claim must include the source URL in your response.
+4. **STRUCTURE CLEARLY**: Display your findings in an ordered, structured manner using day-to-day English.
+5. **BE HELPFUL**: If the evidence contains relevant information (even if not a complete answer), present it. Do not say "I couldn't find this" if there is ANY relevant evidence.
+6. **HONEST ABOUT GAPS**: Only if you have NO relevant evidence at all should you indicate what information might be missing. But first, thoroughly review ALL the evidence provided.
+
+CRITICAL: The evidence has been extracted from official UA sources. Your job is to USE this evidence to help the student. Do not dismiss it. Analyze it carefully and extract all relevant information."""
+
+    # Build user prompt
+    user_prompt = f"""Question: {query}
+
+EVIDENCE FROM UA SOURCES (from {evidence_count} snippets across {source_count} sources):
+{evidence_text}
+
+SOURCE URLs:
+{chr(10).join(f"- {url}" for url in source_urls[:15])}
+
+INSTRUCTIONS:
+1. Carefully review ALL the evidence provided above
+2. Extract and present ALL information from the evidence that is relevant to the question
+3. Structure your answer clearly with bullet points or numbered lists where appropriate
+4. Include source URLs for each key piece of information
+5. If the evidence contains relevant information (even if partial), present it - do not say "I couldn't find this"
+6. Only indicate missing information if you have reviewed ALL evidence and found NOTHING relevant
+
+Now provide a comprehensive answer based on the evidence above:"""
+    
+    try:
+        headers = {
+            'Authorization': f'Bearer {api_key}',
+            'Content-Type': 'application/json',
+            'HTTP-Referer': 'https://github.com',  # Optional: for OpenRouter analytics
+            'X-Title': 'UA Student Navigator'  # Optional: for OpenRouter analytics
+        }
+        
+        payload = {
+            'model': model,
+            'messages': [
+                {'role': 'system', 'content': system_prompt},
+                {'role': 'user', 'content': user_prompt}
+            ],
+            'temperature': 0.7,
+            'max_tokens': 1500  # Reduced to prevent timeouts
+        }
+        
+        # Check rate limit before making request
+        rate_tracker = get_rate_limit_tracker()
+        can_request, rate_limit_msg = rate_tracker.can_make_request(model, api_key)
+        
+        if not can_request:
+            # Return partial answer if we have partial evidence
+            if partial_evidence:
+                partial_answer = _generate_partial_answer(query, partial_evidence)
+                return partial_answer, True, rate_limit_msg
+            return "Rate limit reached. Please wait a moment and try again.", True, rate_limit_msg
+        
+        # Record request
+        rate_tracker.record_request(model)
+        
+        # Use longer timeout and retry logic for reliability
+        max_retries = 2
+        timeout = 120.0  # Increased to 120 seconds for large models
+        
+        for attempt in range(max_retries):
+            try:
+                with httpx.Client(timeout=timeout) as client:
+                    response = client.post(
+                        'https://openrouter.ai/api/v1/chat/completions',
+                        json=payload,
+                        headers=headers
+                    )
+                    
+                    # Check for rate limit (429)
+                    if response.status_code == 429:
+                        error_data = {}
+                        try:
+                            error_data = response.json()
+                        except:
+                            pass
+                        error_msg = error_data.get('error', {}).get('message', 'Rate limit exceeded')
+                        
+                        # Exponential backoff
+                        wait_time = (2 ** attempt) * 2  # 2s, 4s
+                        if attempt < max_retries - 1:
+                            time.sleep(wait_time)
+                            continue
+                        else:
+                            # Return partial answer if available
+                            if partial_evidence:
+                                partial_answer = _generate_partial_answer(query, partial_evidence)
+                                return partial_answer, True, f"Rate limit reached: {error_msg}"
+                            return "Rate limit reached. Please wait a moment and try again.", True, error_msg
+                    
+                    response.raise_for_status()
+                    result = response.json()
+                    
+                    # Extract answer
+                    if 'choices' in result and len(result['choices']) > 0:
+                        answer = result['choices'][0]['message']['content']
+                        return answer.strip(), False, None
+                    else:
+                        return f"Error: Unexpected response format from OpenRouter", False, None
+            
+            except httpx.TimeoutException:
+                if attempt < max_retries - 1:
+                    # Reduce evidence on retry to speed up
+                    if attempt == 1:
+                        evidence_text = "\n\n".join([
+                            f"[Source: {snippet.get('source_url', 'unknown')}]\n{snippet.get('text', '')}"
+                            for snippet in evidence_snippets[:10]  # Reduce to 10 on retry
+                        ])
+                        user_prompt = f"""Question: {query}
+
+EVIDENCE FROM UA SOURCES (from {min(len(evidence_snippets), 10)} snippets):
+{evidence_text}
+
+SOURCE URLs:
+{chr(10).join(f"- {url}" for url in source_urls[:10])}
+
+INSTRUCTIONS:
+1. Review the evidence above
+2. Extract and present ALL relevant information
+3. Include source URLs for key claims
+4. If evidence contains relevant information, present it
+
+Provide a comprehensive answer:"""
+                        payload['messages'][1]['content'] = user_prompt
+                    continue
+                else:
+                    # Return partial answer if available
+                    if partial_evidence:
+                        partial_answer = _generate_partial_answer(query, partial_evidence)
+                        return partial_answer, False, "Request timed out. Showing partial results."
+                    return "Error: Request to OpenRouter timed out after multiple attempts. The model may be slow or overloaded. Try a different model.", False, None
+    
+    except httpx.TimeoutException:
+        if partial_evidence:
+            partial_answer = _generate_partial_answer(query, partial_evidence)
+            return partial_answer, False, "Request timed out. Showing partial results."
+        return "Error: Request to OpenRouter timed out. Try a different model or reduce the amount of evidence.", False, None
+    except httpx.HTTPStatusError as e:
+        error_data = {}
+        try:
+            error_data = e.response.json()
+        except:
+            pass
+        
+        error_msg = error_data.get('error', {}).get('message', f'HTTP {e.response.status_code}')
+        if 'rate' in error_msg.lower() or '429' in str(e.response.status_code):
+            # Return partial answer if available
+            if partial_evidence:
+                partial_answer = _generate_partial_answer(query, partial_evidence)
+                return partial_answer, True, f"Rate limit reached: {error_msg}"
+            return f"Rate limit reached. Please try a different model or wait a moment.", True, error_msg
+        return f"Error: OpenRouter API returned: {error_msg}", False, None
+    except Exception as e:
+        if partial_evidence:
+            partial_answer = _generate_partial_answer(query, partial_evidence)
+            return partial_answer, False, f"Error occurred: {str(e)}. Showing partial results."
+        return f"Error generating answer: {str(e)}", False, None
+
+
+def _generate_partial_answer(query: str, partial_evidence: Dict) -> str:
+    """Generate a summary from partial evidence when rate limited."""
+    pages = partial_evidence.get('pages', [])
+    snippets = partial_evidence.get('snippets', [])
+    dates = partial_evidence.get('dates', [])
+    
+    answer_parts = []
+    answer_parts.append("⚠️ **Rate Limit Reached - Partial Results**\n\n")
+    answer_parts.append("I've reached the maximum number of requests for this model. Here's what I found so far:\n\n")
+    
+    if snippets:
+        answer_parts.append("**Key Findings:**\n")
+        for i, snippet in enumerate(snippets[:10], 1):
+            answer_parts.append(f"{i}. {snippet.get('text', '')[:200]}\n")
+            answer_parts.append(f"   *Source: [{snippet.get('source_title', 'Link')}]({snippet.get('source_url', '')})*\n\n")
+    
+    if dates:
+        answer_parts.append("\n**Important Dates:**\n")
+        for date_info in dates[:5]:
+            answer_parts.append(f"- **{date_info.get('date', 'Unknown')}**: {date_info.get('context', '')[:150]}\n")
+            answer_parts.append(f"  *Source: [{date_info.get('source_title', 'Link')}]({date_info.get('source_url', '')})*\n")
+    
+    if pages:
+        answer_parts.append("\n\n**Sources Analyzed:**\n")
+        for i, page in enumerate(pages[:10], 1):
+            source_indicator = " (via Wayback Machine)" if page.get('source') == 'wayback_machine' else ""
+            answer_parts.append(f"{i}. [{page.get('title', 'No title')}]({page.get('url', '')}){source_indicator}\n")
+    
+    answer_parts.append("\n\n**Note:** To see complete results, please wait a moment and try again, or select a different model.")
+    
+    return "".join(answer_parts)
+
+
+def generate_answer(
+    query: str,
+    evidence_snippets: List[Dict[str, str]],
+    source_urls: List[str],
+    ollama_host: Optional[str] = None,
+    model: Optional[str] = None
+) -> str:
+    """
+    Generate answer using Ollama LLM based on evidence (for local models).
+    
+    Args:
+        query: User question
+        evidence_snippets: List of dicts with 'text' and 'source_url' keys
+        source_urls: List of source URLs
+        ollama_host: Ollama host URL
+        model: Model name
+    
+    Returns:
+        Generated answer text
+    """
+    if ollama_host is None:
+        ollama_host = os.getenv('OLLAMA_HOST', 'http://localhost:11434')
+    
+    if model is None:
+        model = os.getenv('OLLAMA_MODEL', 'qwen2.5:7b')
+    
+    # Build evidence text
+    evidence_text = "\n\n".join([
+        f"[Source: {snippet.get('source_url', 'unknown')}]\n{snippet.get('text', '')}"
+        for snippet in evidence_snippets[:20]
+    ])
+    
+    evidence_count = len(evidence_snippets)
+    source_count = len(source_urls)
+    
+    # Build system prompt
+    system_prompt = """You are a professional research assistant for University of Arizona students. Your job is to ensure students obtain comprehensive information about the University of Arizona and everything it has to offer, including but not limited to:
+
+1. Student life
+2. Academic and research opportunities
+3. Clubs and student organizations
+4. Off-campus resources provided by the University of Arizona
+5. Campus resources
+6. Staff, professors, and entertainment spots
+7. Updates on the University of Arizona that affect students
+
+Your mission is to make student life and student research much easier. You have access to research tools (SearXNG) that allow you to search across arizona.edu domains and read through official UA websites.
+
+WORK STRUCTURE:
+• The user asks a question
+• Research has been conducted across arizona.edu domains
+• Websites and URLs have been scraped and read through
+• Key facts and dates matching the user's query have been extracted
+• You now have evidence from official UA sources
+
+YOUR RESPONSIBILITIES:
+1. **USE THE EVIDENCE PROVIDED**: The evidence below comes from official UA webpages. You MUST use this evidence to answer the question.
+2. **BE COMPREHENSIVE**: Extract and present all relevant information from the evidence that answers or relates to the user's question.
+3. **CITE SOURCES**: Every key claim must include the source URL in your response.
+4. **STRUCTURE CLEARLY**: Display your findings in an ordered, structured manner using day-to-day English.
+5. **BE HELPFUL**: If the evidence contains relevant information (even if not a complete answer), present it. Do not say "I couldn't find this" if there is ANY relevant evidence.
+6. **HONEST ABOUT GAPS**: Only if you have NO relevant evidence at all should you indicate what information might be missing. But first, thoroughly review ALL the evidence provided.
+
+CRITICAL: The evidence has been extracted from official UA sources. Your job is to USE this evidence to help the student. Do not dismiss it. Analyze it carefully and extract all relevant information."""
+
+    # Build user prompt
+    user_prompt = f"""Question: {query}
+
+EVIDENCE FROM UA SOURCES (from {evidence_count} snippets across {source_count} sources):
+{evidence_text}
+
+SOURCE URLs:
+{chr(10).join(f"- {url}" for url in source_urls[:15])}
+
+INSTRUCTIONS:
+1. Carefully review ALL the evidence provided above
+2. Extract and present ALL information from the evidence that is relevant to the question
+3. Structure your answer clearly with bullet points or numbered lists where appropriate
+4. Include source URLs for each key piece of information
+5. If the evidence contains relevant information (even if partial), present it - do not say "I couldn't find this"
+6. Only indicate missing information if you have reviewed ALL evidence and found NOTHING relevant
+
+Now provide a comprehensive answer based on the evidence above:"""
+    
+    try:
+        with httpx.Client(timeout=60.0) as client:
+            response = client.post(
+                f"{ollama_host.rstrip('/')}/api/chat",
+                json={
+                    "model": model,
+                    "messages": [
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt}
+                    ],
+                    "stream": False
+                }
+            )
+            response.raise_for_status()
+            result = response.json()
+            answer = result.get('message', {}).get('content', '')
+            return answer.strip()
+    
+    except httpx.TimeoutException:
+        return "Error: Request to Ollama timed out. Please try again."
+    except httpx.HTTPStatusError as e:
+        return f"Error: Ollama API returned status {e.response.status_code}. Please check your Ollama setup."
+    except Exception as e:
+        return f"Error generating answer: {str(e)}"
+
+
+# Keep Ollama functions for backward compatibility
+def check_ollama_available(ollama_host: Optional[str] = None) -> Tuple[bool, Optional[str]]:
+    """Check if Ollama is running and accessible."""
+    if ollama_host is None:
+        ollama_host = os.getenv('OLLAMA_HOST', 'http://localhost:11434')
+    
+    try:
+        with httpx.Client(timeout=5.0) as client:
+            response = client.get(f"{ollama_host.rstrip('/')}/api/tags")
+            response.raise_for_status()
+        return True, None
+    except Exception as e:
+        return False, f"Ollama is not running or not accessible at {ollama_host}: {str(e)}"
+
+
+def check_model_exists(model: str, ollama_host: Optional[str] = None) -> Tuple[bool, Optional[str], Optional[str]]:
+    """Check if the specified Ollama model exists locally."""
+    if ollama_host is None:
+        ollama_host = os.getenv('OLLAMA_HOST', 'http://localhost:11434')
+    
+    try:
+        with httpx.Client(timeout=10.0) as client:
+            response = client.get(f"{ollama_host.rstrip('/')}/api/tags")
+            response.raise_for_status()
+            models = response.json().get('models', [])
+            model_names = [m.get('name', '') for m in models]
+            
+            if model in model_names or any(m.startswith(model) for m in model_names):
+                return True, None, model
+            
+            # Suggest alternatives
+            preferred_models = ['qwen2.5:7b', 'gemma3:latest', 'phi3:latest', 'llama3.2:3b']
+            suggested = None
+            for preferred in preferred_models:
+                if preferred in model_names or any(m.startswith(preferred.split(':')[0]) for m in model_names):
+                    for m in model_names:
+                        if m == preferred or m.startswith(preferred.split(':')[0] + ':'):
+                            suggested = m
+                            break
+                    if suggested:
+                        break
+            
+            if not suggested and model_names:
+                suggested = model_names[0]
+            
+            error_msg = f"Model '{model}' not found."
+            if suggested:
+                error_msg += f" Suggested: '{suggested}' (run: ollama pull {suggested})"
+            else:
+                error_msg += f" Available: {', '.join(model_names[:5])}"
+            
+            return False, error_msg, suggested
+    except Exception as e:
+        return False, f"Error checking model: {str(e)}", None
+

rank.py

@@ -0,0 +1,85 @@
+"""
+Rank pages by relevance to query.
+"""
+from typing import List, Dict
+import re
+
+
+def calculate_relevance_score(page: Dict[str, str], query: str) -> float:
+    """Calculate relevance score for a page based on query."""
+    query_lower = query.lower()
+    query_words = [w.lower() for w in query.split() if len(w) > 2]
+    
+    if not query_words:
+        return 0.0
+    
+    title = page.get('title', '').lower()
+    content = page.get('content', '').lower()
+    url = page.get('url', '').lower()
+    
+    score = 0.0
+    
+    for word in query_words:
+        if word in title:
+            score += 10.0
+        if query_lower in title:
+            score += 20.0
+    
+    for word in query_words:
+        if word in url:
+            score += 5.0
+    
+    content_matches = 0
+    for word in query_words:
+        count = content.count(word)
+        content_matches += min(count, 10)
+    
+    score += content_matches * 1.0
+    
+    if query_lower in content:
+        score += 15.0
+    
+    content_length = len(content)
+    if content_length > 0:
+        score = score / (1 + content_length / 10000)
+    
+    return score
+
+
+def rank_pages(pages: List[Dict[str, str]], query: str) -> List[Dict[str, str]]:
+    """Rank pages by relevance to query."""
+    scored_pages = []
+    for page in pages:
+        score = calculate_relevance_score(page, query)
+        page_copy = page.copy()
+        page_copy['relevance_score'] = score
+        scored_pages.append(page_copy)
+    
+    scored_pages.sort(key=lambda x: x['relevance_score'], reverse=True)
+    return scored_pages
+
+
+def rank_snippets(snippets: List[Dict[str, str]], query: str) -> List[Dict[str, str]]:
+    """Rank snippets by relevance."""
+    query_lower = query.lower()
+    query_words = [w.lower() for w in query.split() if len(w) > 2]
+    
+    scored_snippets = []
+    for snippet in snippets:
+        text_lower = snippet.get('text', '').lower()
+        score = 0.0
+        
+        for word in query_words:
+            if word in text_lower:
+                score += 1.0
+        
+        if query_lower in text_lower:
+            score += 5.0
+        
+        snippet_copy = snippet.copy()
+        snippet_copy['relevance_score'] = score
+        scored_snippets.append(snippet_copy)
+    
+    scored_snippets.sort(key=lambda x: x['relevance_score'], reverse=True)
+    return scored_snippets
+

rate_limit.py

@@ -0,0 +1,126 @@
+"""
+Rate limit tracking and management for OpenRouter API.
+"""
+import httpx
+import time
+from typing import Dict, Optional, Tuple
+from collections import defaultdict
+from datetime import datetime, timedelta
+
+
+class RateLimitTracker:
+    """Track rate limits per model and manage request throttling."""
+    
+    def __init__(self):
+        self.request_counts: Dict[str, list] = defaultdict(list)  # model -> [timestamps]
+        self.rate_limit_info: Dict[str, Dict] = {}  # model -> rate limit info
+        self.last_check: Optional[datetime] = None
+        self.global_rate_limit: Optional[Dict] = None
+    
+    def get_rate_limits(self, api_key: str) -> Dict:
+        """Get rate limit information from OpenRouter API."""
+        # Cache for 5 minutes
+        if self.global_rate_limit and self.last_check:
+            if datetime.now() - self.last_check < timedelta(minutes=5):
+                return self.global_rate_limit
+        
+        try:
+            headers = {'Authorization': f'Bearer {api_key}'}
+            with httpx.Client(timeout=10.0) as client:
+                response = client.get('https://openrouter.ai/api/v1/key', headers=headers)
+                if response.status_code == 200:
+                    data = response.json()
+                    self.global_rate_limit = data
+                    self.last_check = datetime.now()
+                    return data
+        except Exception as e:
+            print(f"Error fetching rate limits: {e}")
+        
+        # Default conservative limits if API fails
+        return {
+            'limit': {
+                'requests_per_minute': 20,
+                'requests_per_day': 1000
+            }
+        }
+    
+    def get_model_rate_limit(self, model: str, api_key: str) -> Dict:
+        """Get rate limit for a specific model."""
+        # Free models typically have lower limits
+        # Conservative defaults based on common free tier limits
+        default_limits = {
+            'requests_per_minute': 10,
+            'requests_per_day': 200,
+            'tokens_per_minute': 100000
+        }
+        
+        # Some models may have different limits
+        model_specific = {
+            'xiaomi/mimo-v2-flash:free': {'requests_per_minute': 15, 'requests_per_day': 300},
+            'deepseek/deepseek-r1-0528:free': {'requests_per_minute': 10, 'requests_per_day': 200},
+            'openai/gpt-oss-20b:free': {'requests_per_minute': 5, 'requests_per_day': 100},
+            'nvidia/nemotron-3-nano-30b-a3b:free': {'requests_per_minute': 10, 'requests_per_day': 200},
+        }
+        
+        if model in model_specific:
+            return {**default_limits, **model_specific[model]}
+        
+        return default_limits
+    
+    def can_make_request(self, model: str, api_key: str) -> Tuple[bool, Optional[str]]:
+        """Check if we can make a request without hitting rate limit."""
+        limits = self.get_model_rate_limit(model, api_key)
+        now = datetime.now()
+        
+        # Clean old timestamps (older than 1 minute)
+        cutoff = now - timedelta(minutes=1)
+        self.request_counts[model] = [
+            ts for ts in self.request_counts[model] 
+            if ts > cutoff
+        ]
+        
+        # Check per-minute limit
+        requests_last_minute = len(self.request_counts[model])
+        if requests_last_minute >= limits.get('requests_per_minute', 10):
+            wait_time = 60 - (now - self.request_counts[model][0]).total_seconds()
+            return False, f"Rate limit reached: {requests_last_minute}/{limits.get('requests_per_minute')} requests per minute. Wait {int(wait_time)}s"
+        
+        return True, None
+    
+    def record_request(self, model: str):
+        """Record that a request was made."""
+        self.request_counts[model].append(datetime.now())
+    
+    def get_remaining_requests(self, model: str, api_key: str) -> int:
+        """Get remaining requests for this minute."""
+        limits = self.get_model_rate_limit(model, api_key)
+        now = datetime.now()
+        
+        # Clean old timestamps
+        cutoff = now - timedelta(minutes=1)
+        self.request_counts[model] = [
+            ts for ts in self.request_counts[model] 
+            if ts > cutoff
+        ]
+        
+        requests_last_minute = len(self.request_counts[model])
+        max_per_minute = limits.get('requests_per_minute', 10)
+        return max(0, max_per_minute - requests_last_minute)
+    
+    def calculate_max_searches(self, model: str, api_key: str) -> int:
+        """Calculate maximum searches based on rate limits."""
+        remaining = self.get_remaining_requests(model, api_key)
+        # Reserve 1 request for final answer generation
+        # Use 80% of remaining to be safe
+        max_searches = max(1, int(remaining * 0.8) - 1)
+        return min(max_searches, 10)  # Cap at 10 for practical purposes
+
+
+# Global rate limit tracker instance
+_rate_limit_tracker = RateLimitTracker()
+
+
+def get_rate_limit_tracker() -> RateLimitTracker:
+    """Get the global rate limit tracker."""
+    return _rate_limit_tracker
+

requirements.txt

@@ -0,0 +1,6 @@
+streamlit>=1.28.0
+httpx>=0.25.0
+beautifulsoup4>=4.12.0
+lxml>=4.9.0
+python-dotenv>=1.0.0
+

search.py

@@ -0,0 +1,259 @@
+"""
+SearXNG integration for searching UA domains with DuckDuckGo fallback.
+"""
+import httpx
+import os
+from typing import List, Dict, Optional
+from urllib.parse import urlparse, quote
+import re
+from bs4 import BeautifulSoup
+
+
+def is_ua_domain(url: str) -> bool:
+    """Check if URL belongs to UA domain (arizona.edu or subdomains)."""
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        return domain.endswith('.arizona.edu') or domain == 'arizona.edu'
+    except Exception:
+        return False
+
+
+def ua_search(query: str, max_results: int = 10, searxng_url: Optional[str] = None) -> List[Dict[str, str]]:
+    """
+    Search using SearXNG, filtering to UA domains only.
+    Tries multiple SearXNG instances as fallback.
+    
+    Args:
+        query: Search query
+        max_results: Maximum number of results to return
+        searxng_url: SearXNG instance URL (defaults to env var)
+    
+    Returns:
+        List of dicts with 'title', 'url', 'snippet' keys, filtered to UA domains
+    """
+    if searxng_url is None:
+        searxng_url = os.getenv('SEARXNG_URL', 'http://localhost:8080')
+    
+    # Enhance query to prefer UA domains
+    enhanced_query = f"site:arizona.edu {query}"
+    
+    # List of SearXNG instances to try (fallback order)
+    searxng_instances = [
+        searxng_url,
+        'https://searx.be',
+        'https://search.sapti.me',
+        'https://searx.tiekoetter.com',
+    ]
+    
+    # Remove duplicates while preserving order
+    seen = set()
+    unique_instances = []
+    for instance in searxng_instances:
+        if instance not in seen:
+            seen.add(instance)
+            unique_instances.append(instance)
+    
+    last_error = None
+    
+    for instance_url in unique_instances:
+        try:
+            # Proper headers to avoid bot detection
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7',
+                'Accept-Language': 'en-US,en;q=0.9',
+                'Referer': instance_url.rstrip('/'),
+                'Origin': instance_url.rstrip('/'),
+            }
+            
+            # For local SearXNG, use HTML parsing since JSON API is blocked
+            client = httpx.Client(timeout=30.0, follow_redirects=True, headers=headers)
+            
+            # Visit main page first to establish session
+            try:
+                client.get(instance_url.rstrip('/'), timeout=5.0)
+            except:
+                pass
+            
+            # Try JSON API first
+            api_url = f"{instance_url.rstrip('/')}/search"
+            params = {'q': enhanced_query, 'format': 'json'}
+            
+            try:
+                response = client.get(api_url, params=params)
+                if response.status_code == 200:
+                    try:
+                        data = response.json()
+                        client.close()
+                        # Success with JSON, process it below
+                    except:
+                        # Not JSON, fall through to HTML parsing
+                        raise ValueError("Not JSON")
+                else:
+                    # 403 or other error, try HTML parsing
+                    raise ValueError(f"HTTP {response.status_code}")
+            except:
+                # JSON API failed, try HTML parsing
+                try:
+                    # Use POST method for HTML search (more likely to work)
+                    html_params = {'q': enhanced_query}
+                    response = client.post(api_url, data=html_params, timeout=30.0)
+                    
+                    if response.status_code == 200:
+                        # Parse HTML results
+                        soup = BeautifulSoup(response.text, 'html.parser')
+                        data = {'results': []}
+                        
+                        # SearXNG uses <article> tags for results
+                        articles = soup.find_all('article')
+                        
+                        for article in articles:
+                            try:
+                                # Extract URL from link in article
+                                link = article.find('a', href=True)
+                                if not link:
+                                    # Try h3 > a structure
+                                    h3 = article.find('h3')
+                                    if h3:
+                                        link = h3.find('a', href=True)
+                                
+                                if link:
+                                    url = link.get('href', '')
+                                    
+                                    # Extract title
+                                    title = link.get_text(strip=True)
+                                    if not title:
+                                        h3 = article.find('h3')
+                                        if h3:
+                                            title = h3.get_text(strip=True)
+                                    
+                                    # Extract snippet/content
+                                    snippet_elem = article.find('p')
+                                    if not snippet_elem:
+                                        snippet_elem = article.find('div', class_='content')
+                                    if not snippet_elem:
+                                        snippet_elem = article.find('div')
+                                    
+                                    snippet = snippet_elem.get_text(strip=True) if snippet_elem else ''
+                                    
+                                    if url:
+                                        data['results'].append({
+                                            'url': url,
+                                            'title': title or 'No title',
+                                            'content': snippet
+                                        })
+                            except Exception as e:
+                                continue
+                        
+                        client.close()
+                        # Success with HTML parsing
+                    else:
+                        client.close()
+                        last_error = f"Instance {instance_url} returned HTTP {response.status_code}"
+                        continue
+                except Exception as e:
+                    client.close()
+                    last_error = f"Instance {instance_url} error: {str(e)}"
+                    continue
+            
+            results = []
+            seen_urls = set()
+            
+            # Extract results from SearXNG response
+            search_results = data.get('results', [])
+            
+            for result in search_results:
+                url = result.get('url', '') or result.get('link', '')
+                if not url:
+                    continue
+                
+                # Filter to UA domains only
+                if not is_ua_domain(url):
+                    continue
+                
+                # Deduplicate
+                if url in seen_urls:
+                    continue
+                seen_urls.add(url)
+                
+                title = result.get('title', '') or result.get('name', 'No title')
+                snippet = result.get('content', '') or result.get('snippet', '') or result.get('description', '')
+                
+                results.append({
+                    'title': title,
+                    'url': url,
+                    'snippet': snippet[:500]
+                })
+                
+                if len(results) >= max_results:
+                    break
+            
+            # If we got results, return them
+            if results:
+                return results
+        
+        except httpx.TimeoutException:
+            last_error = f"Instance {instance_url} timed out"
+            continue
+        except httpx.RequestError as e:
+            last_error = f"Instance {instance_url} request error: {str(e)}"
+            continue
+        except httpx.HTTPStatusError as e:
+            last_error = f"Instance {instance_url} HTTP error: {e.response.status_code}"
+            continue
+        except Exception as e:
+            last_error = f"Instance {instance_url} error: {str(e)}"
+            continue
+    
+    # If all SearXNG instances failed, try DuckDuckGo as fallback
+    print(f"SearXNG search failed on all instances. Last error: {last_error}")
+    print("Trying DuckDuckGo as fallback...")
+    
+    return duckduckgo_fallback_search(enhanced_query, max_results)
+
+
+def duckduckgo_fallback_search(query: str, max_results: int = 10) -> List[Dict[str, str]]:
+    """Fallback search using DuckDuckGo HTML scraping."""
+    try:
+        search_url = f"https://html.duckduckgo.com/html/?q={quote(query)}"
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+        
+        with httpx.Client(timeout=15.0, follow_redirects=True) as client:
+            response = client.get(search_url, headers=headers)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.text, 'html.parser')
+            results = []
+            seen_urls = set()
+            
+            for result in soup.find_all('div', class_='result'):
+                try:
+                    link_elem = result.find('a', class_='result__a')
+                    if not link_elem:
+                        continue
+                    url = link_elem.get('href', '')
+                    if not url or not is_ua_domain(url):
+                        continue
+                    if url in seen_urls:
+                        continue
+                    seen_urls.add(url)
+                    
+                    title = link_elem.get_text(strip=True) or 'No title'
+                    snippet_elem = result.find('a', class_='result__snippet')
+                    snippet = snippet_elem.get_text(strip=True) if snippet_elem else ''
+                    
+                    results.append({'title': title, 'url': url, 'snippet': snippet[:500]})
+                    if len(results) >= max_results:
+                        break
+                except Exception:
+                    continue
+            
+            if results:
+                print(f"✓ DuckDuckGo fallback found {len(results)} results")
+                return results
+    except Exception as e:
+        print(f"DuckDuckGo fallback also failed: {e}")
+    
+    return []
+