Initial Spring 2026 deployment

.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+
+# Environment variables
+.env
+.env.local
+environment.yml
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Data files (upload separately or use external hosting)
+*.csv
+*.parquet
+*.pickle
+*.pkl
+data/raw/
+data/processed/
+
+# Logs
+*.log
+logs/
+
+# Database
+*.db
+*.sqlite
+
+# OS
+.DS_Store
+Thumbs.db
+
+# HuggingFace specific
+flagged/

README.md

@@ -1,20 +1,115 @@
 ---
 title: BPL RAG Spring 2026
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
+emoji: 📚
+colorFrom: blue
+colorTo: green
+sdk: streamlit
+sdk_version: 1.32.0
+app_file: app.py
 pinned: false
-short_description: Ask questions in plain English to discover historical photos
 license: mit
 ---
 
-# Welcome to Streamlit!
+# Boston Public Library - RAG Search System (Spring 2026)
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+Natural language search for the BPL Digital Commonwealth collection using Retrieval-Augmented Generation.
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+## 🎯 Project Overview
+
+This system allows users to search through the Boston Public Library's digital collections using natural language queries. Built as part of the BU Spark! DS549 course in Spring 2026.
+
+### Key Features
+
+- **Semantic Search**: Natural language queries across 445K+ BPL items
+- **Full-Text Search**: Search within document content, not just metadata
+- **Metadata Filtering**: Time-based, location-based, and type-based filtering
+- **Evaluation Framework**: Built-in testing with gold-standard datasets
+- **PostgreSQL + pgvector**: Scalable vector database backend
+
+## 🚀 Quick Start
+
+1. Enter your query in natural language (e.g., "What were important events in Boston in 1919?")
+2. View retrieved documents with relevance scores
+3. Read AI-generated contextual explanations
+
+## 📊 Example Queries
+
+- "Find pictures of JFK's house on Cape Cod"
+- "Are there any maps of Worcester, MA from the 18th century?"
+- "Show me depictions of indigenous Americans"
+- "What were important historical events in Boston in 1919?"
+
+## 🛠️ Technical Stack
+
+- **Frontend**: Streamlit
+- **Database**: PostgreSQL with pgvector extension
+- **Embeddings**: Sentence Transformers
+- **LLM**: OpenAI GPT-4 / Anthropic Claude
+- **Retrieval**: BM25 + Vector Search with metadata filtering
+- **Evaluation**: DeepEval framework
+
+## 📁 Project Structure
+
+```
+current_spring2026/
+├── app.py                      # Main Streamlit application
+├── pipeline.py                 # RAG pipeline orchestration
+├── config.py                   # Configuration management
+├── database/                   # Database connection & queries
+├── embedding/                  # Vector embeddings
+├── retrieval/                  # Document retrieval logic
+├── generation/                 # LLM response generation
+├── evaluation/                 # Testing & metrics
+├── ingestion/                  # Data processing pipeline
+└── scripts/                    # Utility scripts
+```
+
+## 🔐 Environment Variables
+
+Required secrets (set in Space Settings → Repository secrets):
+
+```
+OPENAI_API_KEY=your_openai_key
+DATABASE_URL=postgresql://user:pass@host:port/dbname
+```
+
+## 📈 Spring 2026 Improvements
+
+Building on Fall 2025 work, this version adds:
+
+- ✅ Full-text document search (not just metadata)
+- ✅ Gold-standard evaluation dataset
+- ✅ Structured logging of queries and metrics
+- ✅ Improved retrieval metrics (Precision, Recall, MRR)
+- ✅ Enhanced UI with developer debug view
+
+## 📚 Data
+
+- **Source**: Digital Commonwealth (BPL subset)
+- **Records**: ~445,000 items
+- **Types**: Photographs, maps, newspapers, manuscripts, books
+- **Full-text**: 210K items with OCR (~1.5M pages)
+
+## 👥 Team
+
+**Spring 2026 Spark! Team**
+- Boston University Data Science (DS549)
+- Client: Eben English, Boston Public Library
+
+**Previous Semesters**
+- Fall 2025: Infrastructure & pgvector migration
+- Fall 2024: Initial RAG prototype
+
+## 📄 License
+
+MIT License
+
+## 🔗 Links
+
+- [Digital Commonwealth](https://www.digitalcommonwealth.org/)
+- [BPL Digital Repository](https://www.digitalcommonwealth.org/institutions/boston-public-library)
+- [Project Documentation](https://github.com/your-repo-link)
+
+---
+
+*Built with ❤️ by BU Spark! for Boston Public Library*

app.py

@@ -0,0 +1,492 @@
+import streamlit as st
+import sys
+import os
+import html
+
+# Add project root to path so we can import pipeline modules
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from pipeline import run_query, PipelineResult
+from retrieval.retriever import RetrievedDocument
+
+# ── Page config ──────────────────────────────────────────────────────────────
+st.set_page_config(
+    page_title="Digital Commonwealth · BPL Search",
+    page_icon="📚",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+# ── Custom CSS ────────────────────────────────────────────────────────────────
+st.markdown("""
+<style>
+@import url('https://fonts.googleapis.com/css2?family=Playfair+Display:ital,wght@0,400;0,600;1,400&family=Source+Sans+3:wght@300;400;500;600&display=swap');
+
+/* ── Root palette ── */
+:root {
+    --cream:    #F7F3EC;
+    --ink:      #1A1410;
+    --sepia:    #7A5C3A;
+    --gold:     #C8973A;
+    --rust:     #A63D2F;
+    --muted:    #8A7B6A;
+    --border:   #D9CFC2;
+    --card-bg:  #FFFDF9;
+    --tag-bg:   #EDE5D8;
+}
+
+html, body, [class*="css"] {
+    font-family: 'Source Sans 3', sans-serif;
+    background-color: var(--cream);
+    color: var(--ink);
+}
+
+/* ── Hide default Streamlit chrome ── */
+#MainMenu, footer, header { visibility: hidden; }
+.block-container { padding-top: 2rem; padding-bottom: 3rem; max-width: 960px; }
+
+/* ── Masthead ── */
+.masthead {
+    text-align: center;
+    padding: 3.5rem 1rem 2rem;
+    border-bottom: 2px solid var(--border);
+    margin-bottom: 2.5rem;
+}
+.masthead-eyebrow {
+    font-family: 'Source Sans 3', sans-serif;
+    font-size: 0.72rem;
+    font-weight: 600;
+    letter-spacing: 0.22em;
+    text-transform: uppercase;
+    color: var(--sepia);
+    margin-bottom: 0.6rem;
+}
+.masthead-title {
+    font-family: 'Playfair Display', Georgia, serif;
+    font-size: 3rem;
+    font-weight: 400;
+    color: var(--ink);
+    line-height: 1.15;
+    margin: 0 0 0.5rem;
+}
+.masthead-title em {
+    font-style: italic;
+    color: var(--sepia);
+}
+.masthead-sub {
+    font-size: 1rem;
+    color: var(--muted);
+    font-weight: 300;
+    max-width: 540px;
+    margin: 0 auto;
+    line-height: 1.6;
+}
+
+/* ── Search box wrapper ── */
+.search-wrapper {
+    background: var(--card-bg);
+    border: 1.5px solid var(--border);
+    border-radius: 4px;
+    padding: 1.6rem 1.8rem;
+    margin-bottom: 1.2rem;
+    box-shadow: 0 2px 12px rgba(26,20,16,0.06);
+}
+.search-label {
+    font-size: 0.75rem;
+    font-weight: 600;
+    letter-spacing: 0.16em;
+    text-transform: uppercase;
+    color: var(--sepia);
+    margin-bottom: 0.5rem;
+}
+
+/* ── Example query pills ── */
+.pill-row { display: flex; flex-wrap: wrap; gap: 0.5rem; margin-top: 1rem; }
+.pill {
+    background: var(--tag-bg);
+    border: 1px solid var(--border);
+    border-radius: 2px;
+    padding: 0.3rem 0.75rem;
+    font-size: 0.78rem;
+    color: var(--sepia);
+    cursor: pointer;
+    font-style: italic;
+}
+
+/* ── Divider ── */
+.divider {
+    border: none;
+    border-top: 1px solid var(--border);
+    margin: 1.8rem 0;
+}
+
+/* ── Context banner ── */
+.context-banner {
+    background: linear-gradient(135deg, #FDF6E8 0%, #F5EDD8 100%);
+    border-left: 4px solid var(--gold);
+    border-radius: 0 4px 4px 0;
+    padding: 1.2rem 1.5rem;
+    margin-bottom: 2rem;
+    font-size: 0.95rem;
+    line-height: 1.65;
+    color: var(--ink);
+}
+.context-banner strong { color: var(--sepia); }
+
+/* ── Results header ── */
+.results-header {
+    display: flex;
+    align-items: baseline;
+    justify-content: space-between;
+    margin-bottom: 1.2rem;
+}
+.results-count {
+    font-family: 'Playfair Display', serif;
+    font-size: 1.35rem;
+    color: var(--ink);
+}
+.results-count span { color: var(--sepia); font-style: italic; }
+.results-meta {
+    font-size: 0.78rem;
+    color: var(--muted);
+    letter-spacing: 0.06em;
+}
+
+/* ── Result card ── */
+.result-card {
+    background: var(--card-bg);
+    border: 1px solid var(--border);
+    border-radius: 4px;
+    padding: 1.4rem 1.6rem;
+    margin-bottom: 1rem;
+    transition: border-color 0.2s, box-shadow 0.2s;
+    position: relative;
+}
+.result-card:hover {
+    border-color: var(--gold);
+    box-shadow: 0 4px 18px rgba(200,151,58,0.12);
+}
+.card-type-badge {
+    display: inline-block;
+    font-size: 0.65rem;
+    font-weight: 600;
+    letter-spacing: 0.18em;
+    text-transform: uppercase;
+    color: var(--rust);
+    border: 1px solid var(--rust);
+    border-radius: 2px;
+    padding: 0.15rem 0.5rem;
+    margin-bottom: 0.6rem;
+}
+.card-title {
+    font-family: 'Playfair Display', serif;
+    font-size: 1.1rem;
+    font-weight: 600;
+    color: var(--ink);
+    margin-bottom: 0.3rem;
+    line-height: 1.3;
+}
+.card-meta {
+    font-size: 0.8rem;
+    color: var(--muted);
+    margin-bottom: 0.7rem;
+    line-height: 1.5;
+}
+.card-snippet {
+    font-size: 0.88rem;
+    color: #3D3228;
+    line-height: 1.65;
+    margin-bottom: 0.8rem;
+}
+.card-tags { display: flex; flex-wrap: wrap; gap: 0.4rem; }
+.card-tag {
+    background: var(--tag-bg);
+    font-size: 0.72rem;
+    color: var(--sepia);
+    padding: 0.2rem 0.55rem;
+    border-radius: 2px;
+    border: 1px solid var(--border);
+}
+.card-link {
+    font-size: 0.78rem;
+    font-weight: 600;
+    color: var(--rust);
+    letter-spacing: 0.06em;
+    text-transform: uppercase;
+    text-decoration: none;
+    border-bottom: 1px solid transparent;
+}
+.card-link:hover { border-bottom-color: var(--rust); }
+
+/* ── Relevance score bar ── */
+.score-row { display: flex; align-items: center; gap: 0.6rem; margin-top: 0.6rem; }
+.score-label { font-size: 0.7rem; color: var(--muted); letter-spacing: 0.08em; text-transform: uppercase; }
+.score-bar-bg {
+    flex: 1;
+    height: 4px;
+    background: var(--tag-bg);
+    border-radius: 2px;
+    overflow: hidden;
+    max-width: 120px;
+}
+.score-bar-fill {
+    height: 100%;
+    background: linear-gradient(90deg, var(--gold), var(--rust));
+    border-radius: 2px;
+}
+.score-val { font-size: 0.7rem; color: var(--sepia); font-weight: 600; }
+
+/* ── No results ── */
+.no-results {
+    text-align: center;
+    padding: 4rem 2rem;
+    color: var(--muted);
+}
+.no-results-icon { font-size: 3rem; margin-bottom: 1rem; }
+.no-results-title { font-family: 'Playfair Display', serif; font-size: 1.4rem; color: var(--ink); margin-bottom: 0.5rem; }
+
+/* ── Footer ── */
+.bpl-footer {
+    text-align: center;
+    padding: 2.5rem 1rem 1rem;
+    border-top: 1px solid var(--border);
+    margin-top: 3rem;
+    font-size: 0.78rem;
+    color: var(--muted);
+    letter-spacing: 0.05em;
+}
+.bpl-footer strong { color: var(--sepia); }
+</style>
+""", unsafe_allow_html=True)
+
+
+# ── Example queries ───────────────────────────────────────────────────────────
+EXAMPLE_QUERIES = [
+    "What happened in Boston in 1900?",
+    "Find photographs of Greece",
+    "Show me circus posters",
+    "Victorian era correspondence",
+    "Boston Traveler newspaper 1900",
+    "Women's suffrage documents",
+]
+
+import re
+def linkify_citations(text: str, num_docs: int) -> str:
+    """Replace [N] with clickable spans that scroll to result cards."""
+    def replace(match):
+        n = int(match.group(1))
+        if 1 <= n <= num_docs:
+            return (
+                f'<a href="javascript:void(0)" '
+                f'onclick="document.getElementById(\'result-{n}\').scrollIntoView({{behavior:\'smooth\'}})" '
+                f'style="color:var(--rust);font-weight:600;cursor:pointer;">[{n}]</a>'
+            )
+        return match.group(0)
+    import re
+    return re.sub(r'\[(\d+)\]', replace, text)
+
+
+# ── Helper: format a RetrievedDocument into card fields ──────────────────────
+def format_card(doc: RetrievedDocument) -> dict:
+    # Determine document type label
+    topics = doc.topics or []
+    title_lower = (doc.title or "").lower()
+
+    if any(t.lower() in ["photograph", "photography", "photographs"] for t in topics):
+        doc_type = "Photograph"
+    elif any(t.lower() in ["map", "maps", "cartography"] for t in topics):
+        doc_type = "Map"
+    elif any(w in title_lower for w in ["traveler", "globe", "herald", "gazette", "journal", "tribune"]):
+        doc_type = "Newspaper"
+    elif any(t.lower() in ["correspondence", "manuscript", "letter", "papers"] for t in topics):
+        doc_type = "Manuscript"
+    else:
+        doc_type = "Document"
+
+    # Date string
+    if doc.issue_date:
+        date_str = doc.issue_date
+    elif doc.year:
+        date_str = str(doc.year[0])
+    else:
+        date_str = "Date unknown"
+
+    snippet = doc.best_chunk_text[:300] if doc.best_chunk_text else ""
+    tags = list(set((doc.topics or []) + (doc.geography or [])))[:5]
+    if doc.best_chunk_text:
+        # Has chunk text = individual item
+        url = f"https://www.digitalcommonwealth.org/search/commonwealth:{doc.ark_id}"
+    else:
+        # No chunk text = collection-level metadata record
+        url = f"https://www.digitalcommonwealth.org/collections/commonwealth:{doc.ark_id}"
+    
+    thumbnail_url = (
+        f"https://iiif.digitalcommonwealth.org/iiif/2/{doc.exemplary_image_id}/full/400,/0/default.jpg"
+        if doc.exemplary_image_id and doc.exemplary_image_id.strip() else ""
+    )
+    return {
+        "type":       doc_type,
+        "title":      doc.title or "Untitled",
+        "date":       date_str,
+        "collection": doc.institution or "Boston Public Library",
+        "snippet":    snippet,
+        "tags":       tags,
+        "score":      round(doc.final_score, 2),
+        "url":        url,
+        "thumbnail":  thumbnail_url
+    }
+
+
+# ── Session state ─────────────────────────────────────────────────────────────
+if "query" not in st.session_state:
+    st.session_state.query = ""
+if "results" not in st.session_state:
+    st.session_state.results = None
+if "searched" not in st.session_state:
+    st.session_state.searched = False
+if "context" not in st.session_state:
+    st.session_state.context = ""
+if "latency_ms" not in st.session_state:
+    st.session_state.latency_ms = 0
+
+
+# ── Masthead ──────────────────────────────────────────────────────────────────
+st.markdown("""
+<div class="masthead">
+    <div class="masthead-eyebrow">Boston Public Library · Digital Commonwealth</div>
+    <h1 class="masthead-title">Search the <em>Archive</em></h1>
+    <p class="masthead-sub">
+        Ask anything in plain language — explore photographs, maps, newspapers,
+        manuscripts, and more from Massachusetts history.
+    </p>
+</div>
+""", unsafe_allow_html=True)
+
+# ── Search box ────────────────────────────────────────────────────────────────
+st.markdown('<div class="search-wrapper">', unsafe_allow_html=True)
+st.markdown('<div class="search-label">Natural Language Query</div>', unsafe_allow_html=True)
+
+col_input, col_btn = st.columns([5, 1])
+with col_input:
+    query_input = st.text_input(
+        label="query",
+        label_visibility="collapsed",
+        placeholder='e.g. "Find photographs of Boston Harbor from the 1800s"',
+        value=st.session_state.query,
+        key="query_box",
+    )
+with col_btn:
+    search_clicked = st.button("Search →", use_container_width=True, type="primary")
+
+# Example query pills
+st.markdown('<div class="search-label" style="margin-top:1rem;">Try an example</div>', unsafe_allow_html=True)
+pill_cols = st.columns(3)
+for i, example in enumerate(EXAMPLE_QUERIES):
+    with pill_cols[i % 3]:
+        if st.button(f'"{example}"', key=f"pill_{i}", use_container_width=True):
+            st.session_state.query = example
+            st.rerun()
+
+st.markdown('</div>', unsafe_allow_html=True)
+
+# ── Handle search ─────────────────────────────────────────────────────────────
+active_query = st.session_state.query if st.session_state.query else query_input
+
+if search_clicked and query_input.strip():
+    st.session_state.query = query_input.strip()
+    active_query = query_input.strip()
+
+if active_query and (search_clicked or st.session_state.query):
+    with st.spinner("Searching the archive…"):
+        try:
+            result: PipelineResult = run_query(active_query)
+            cards = [format_card(doc) for doc in result.documents]
+            st.session_state.results   = cards
+            st.session_state.context   = result.generation.response
+            st.session_state.latency_ms = result.latency_ms
+            st.session_state.searched  = True
+        except Exception as e:
+            st.error(f"Search failed: {e}")
+            st.session_state.searched = False
+
+# ── Results ───────────────────────────────────────────────────────────────────
+if st.session_state.searched and st.session_state.results is not None:
+    results = st.session_state.results
+    context = st.session_state.context
+    latency = st.session_state.latency_ms
+
+    st.markdown('<hr class="divider">', unsafe_allow_html=True)
+    if results:
+        # Context banner
+        context_with_links = linkify_citations(context, len(results))
+        st.markdown(f"""
+        <div class="context-banner">
+            <strong>About these results —</strong> {context_with_links}
+        </div>
+        """, unsafe_allow_html=True)
+
+        # Results header
+        st.markdown(f"""
+        <div class="results-header">
+            <div class="results-count">
+                Found <span>{len(results)} items</span> for "{st.session_state.query}"
+            </div>
+            <div class="results-meta">Ranked by relevance · {latency}ms · Digital Commonwealth BPL Subset</div>
+        </div>
+        """, unsafe_allow_html=True)
+
+        # Result cards
+        for i, r in enumerate(results, start=1):
+            score_pct = min(int(r["score"] * 100), 100)
+            bar_width = score_pct
+            tags_html = ''.join(f'<span class="card-tag">{t}</span>' for t in r["tags"])
+
+            thumbnail_html = (
+            f'<img src="{r["thumbnail"]}" style="width:100%;max-height:200px;object-fit:cover;border-radius:4px;margin-bottom:0.8rem;" />'
+            if r.get("thumbnail", "").startswith("https://") else "<div></div>"
+            )
+            # print(f"[debug] thumbnail: {r.get('thumbnail')!r}, thumbnail_html: {thumbnail_html!r}")
+
+            safe_snippet = html.escape(r["snippet"])
+            safe_title   = html.escape(r["title"])
+
+            st.markdown(f"""
+            <div class="result-card" id="result-{i+1}">
+                {thumbnail_html}
+                <div class="card-type-badge">{r['type']}</div>
+                <div class="card-title">{safe_title}</div>
+                <div class="card-meta">
+                    {r['date']} &nbsp;·&nbsp; {r['collection']}
+                </div>
+                <div class="card-snippet">{safe_snippet}</div>
+                <div class="card-tags">{tags_html}</div>
+                <div class="score-row">
+                    <span class="score-label">Relevance</span>
+                    <div class="score-bar-bg">
+                        <div class="score-bar-fill" style="width:{bar_width}%"></div>
+                    </div>
+                    <span class="score-val">{score_pct}%</span>
+                    &nbsp;&nbsp;
+                    <a class="card-link" href="{r['url']}" target="_blank">View in Digital Commonwealth ↗</a>
+                </div>
+            </div>
+            """, unsafe_allow_html=True)
+
+    else:
+        st.markdown(f"""
+        <div class="no-results">
+            <div class="no-results-icon">🗂️</div>
+            <div class="no-results-title">No matching materials found</div>
+            <p>Try rephrasing your query, or use one of the example searches above.<br/>
+            The full collection spans photographs, maps, newspapers, manuscripts, and more.</p>
+        </div>
+        """, unsafe_allow_html=True)
+
+# ── Footer ────────────────────────────────────────────────────────────────────
+st.markdown("""
+<div class="bpl-footer">
+    <strong>Boston Public Library</strong> · Digital Commonwealth · BPL RAG Search<br>
+    A natural language search prototype built with Retrieval-Augmented Generation.<br>
+    Results are drawn from digitized items in the BPL subset of Digital Commonwealth.
+</div>
+""", unsafe_allow_html=True)

config.py

@@ -0,0 +1,102 @@
+"""
+Central configuration for the BPL RAG pipeline.
+All tuneable constants live here — change here, affects everywhere.
+
+Updated for HuggingFace deployment - supports both local .env and Streamlit secrets.
+"""
+
+import os
+from typing import Optional
+
+# Try to load from .env file (for local development)
+try:
+    from dotenv import load_dotenv
+    load_dotenv()
+except:
+    pass
+
+def get_secret(key: str, default: Optional[str] = None) -> str:
+    """
+    Get secret from environment or Streamlit secrets.
+    Works for both local development and HuggingFace deployment.
+    
+    Priority:
+    1. Environment variables (from .env or system)
+    2. Streamlit secrets (on HuggingFace)
+    3. Default value
+    """
+    # Try environment variable first
+    value = os.getenv(key)
+    
+    if value:
+        return value
+    
+    # Try Streamlit secrets (available on HuggingFace)
+    try:
+        import streamlit as st
+        if hasattr(st, 'secrets') and key in st.secrets:
+            return st.secrets[key]
+    except Exception:
+        pass
+    
+    # Return default or raise error
+    if default is not None:
+        return default
+    
+    raise ValueError(f"Secret '{key}' not found in environment or Streamlit secrets")
+
+# ── OpenAI ────────────────────────────────────────────────────────────────────
+OPENAI_API_KEY        = get_secret("OPENAI_API_KEY")
+OPENAI_CHAT_MODEL     = "gpt-4o"
+
+# ── BGE Embedding ─────────────────────────────────────────────────────────────
+BGE_MODEL_NAME = "BAAI/bge-m3"
+BGE_DEVICE            = "cpu"  # Changed from "cuda" for HuggingFace CPU deployment
+BGE_BATCH_SIZE        = 32     # Reduced from 128 for CPU
+
+# ── Neo4j / GraphRAG ──────────────────────────────────────────────────────────
+NEO4J_URI      = get_secret("NEO4J_URI", "")
+NEO4J_USER     = get_secret("NEO4J_USER", "neo4j")
+NEO4J_PASSWORD = get_secret("NEO4J_PASSWORD", "")
+
+# GraphRAG is triggered only for content_driven queries
+GRAPH_RAG_ENABLED        = True
+GRAPH_TOP_K              = 100    # max additional docs from graph
+GRAPH_MIN_ENTITY_MATCHES = 1      # min query entities a doc must match
+
+# ── PostgreSQL / pgVector ─────────────────────────────────────────────────────
+PG_HOST               = get_secret("PG_HOST", "localhost")
+PG_PORT               = int(get_secret("PG_PORT", "5432"))
+PG_DB                 = get_secret("PG_DB", "bpl_rag")
+PG_USER               = get_secret("PG_USER", "postgres")
+PG_PASSWORD           = get_secret("PG_PASSWORD", "")
+
+PG_DSN                = (
+    f"postgresql://{PG_USER}:{PG_PASSWORD}@{PG_HOST}:{PG_PORT}/{PG_DB}"
+)
+
+# ── Chunking ──────────────────────────────────────────────────────────────────
+CHUNK_SIZE    = 1024   # was 512 — BGE-M3 handles longer context well
+CHUNK_OVERLAP = 150    # was 100 — proportionally larger overlap
+CHUNK_TOKENIZER       = "cl100k_base"   # tiktoken encoding
+
+# ── Retrieval ─────────────────────────────────────────────────────────────────
+TOP_K_DENSE           = 100     # candidates from vector search before rerank
+TOP_K_BM25            = 100     # candidates from BM25
+TOP_K_FINAL           = 10      # results returned to the user
+RRF_K                 = 60      # RRF constant (standard is 60)
+
+# ── Metadata score blend weight ───────────────────────────────────────────────
+# final_score = CONTENT_WEIGHT * content_rrf + METADATA_WEIGHT * metadata_sim
+CONTENT_WEIGHT        = 0.75
+METADATA_WEIGHT       = 0.25
+
+# ── Ingestion ─────────────────────────────────────────────────────────────────
+MIN_CHAR_COUNT        = 100     # skip records with fewer chars of raw_text
+JSON_DUMP_DIR         = "data/raw"      # folder containing local JSON dumps
+
+# ── Generation ───────────────────────────────────────────────────────────────
+MAX_CONTEXT_CHUNKS    = 5       # how many chunks to pass to GPT-4o
+GENERATION_MAX_TOKENS = 600
+
+MIN_RELEVANCE_SCORE = 0.01   # documents below this are considered irrelevant

database/schema.py

@@ -0,0 +1,163 @@
+"""
+database/schema.py
+
+Creates all pgVector tables from scratch.
+Run once:  python -m database.schema
+"""
+
+import psycopg2
+from psycopg2.extras import RealDictCursor
+from contextlib import contextmanager
+from config import PG_DSN
+
+
+@contextmanager
+def get_conn():
+    conn = psycopg2.connect(PG_DSN)
+    try:
+        yield conn
+        conn.commit()
+    except Exception:
+        conn.rollback()
+        raise
+    finally:
+        conn.close()
+
+
+@contextmanager
+def get_cursor(conn):
+    cur = conn.cursor(cursor_factory=RealDictCursor)
+    try:
+        yield cur
+    finally:
+        cur.close()
+
+
+DDL = """
+CREATE EXTENSION IF NOT EXISTS vector;
+
+-- ── Table 1: documents ───────────────────────────────────────────────────────
+-- One row per source record.
+-- Holds all metadata, dense metadata embedding, and sparse metadata embedding.
+
+CREATE TABLE IF NOT EXISTS documents (
+    id              BIGSERIAL PRIMARY KEY,
+
+    -- Identity
+    ark_id          TEXT UNIQUE NOT NULL,
+    record_id       TEXT,
+    source_url      TEXT,
+    iiif_manifest   TEXT,
+
+    -- Provenance
+    newspaper       TEXT,
+    collection      TEXT,
+    institution     TEXT,
+
+    -- Bibliographic
+    title           TEXT,
+    issue_date      TEXT,
+    date_iso        TEXT[],
+    date_start      TIMESTAMPTZ,
+    year            INT[],
+    publisher       TEXT[],
+    place           TEXT[],
+    language        TEXT[],
+
+    -- Content signals
+    page_count      INT,
+    pages           TEXT[],
+    topics          TEXT[],
+    geography       TEXT[],
+    char_count      INT,
+    genre               TEXT[],
+    abstract            TEXT,
+    exemplary_image_id  TEXT,
+
+    -- Dense metadata embedding (BGE-M3 → 1024 dims)
+    metadata_embedding  VECTOR(1024),
+
+    -- Sparse metadata embedding (two-array format, more efficient than JSONB)
+    sparse_token_ids    INTEGER[],
+    sparse_weights      FLOAT4[],
+
+    -- Tracking
+    ingested_at     TIMESTAMPTZ
+);
+
+-- HNSW index on metadata embedding (faster + smaller than ivfflat)
+CREATE INDEX IF NOT EXISTS idx_documents_meta_emb
+    ON documents
+    USING hnsw (metadata_embedding vector_cosine_ops)
+    WITH (m = 16, ef_construction = 64);
+
+CREATE INDEX IF NOT EXISTS idx_documents_ark      ON documents (ark_id);
+CREATE INDEX IF NOT EXISTS idx_documents_year     ON documents USING GIN (year);
+CREATE INDEX IF NOT EXISTS idx_documents_language ON documents USING GIN (language);
+CREATE INDEX IF NOT EXISTS idx_documents_topics   ON documents USING GIN (topics);
+CREATE INDEX IF NOT EXISTS idx_documents_geo      ON documents USING GIN (geography);
+CREATE INDEX IF NOT EXISTS idx_documents_genre ON documents USING GIN (genre);
+
+
+-- ── Table 2: chunks ──────────────────────────────────────────────────────────
+-- One row per text chunk (512 tokens, 100 overlap).
+-- No tsvector column — lexical search handled by BGE-M3 sparse instead.
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id              BIGSERIAL PRIMARY KEY,
+    document_id     BIGINT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+    ark_id          TEXT NOT NULL,
+
+    chunk_index     INT NOT NULL,
+    chunk_text      TEXT NOT NULL,
+
+    -- Dense full-text embedding (BGE-M3 → 1024 dims)
+    text_embedding  VECTOR(1024),
+
+    -- Sparse full-text embedding (two-array format)
+    sparse_token_ids    INTEGER[],
+    sparse_weights      FLOAT4[]
+);
+
+-- HNSW index on chunk text embedding
+CREATE INDEX IF NOT EXISTS idx_chunks_text_emb
+    ON chunks
+    USING hnsw (text_embedding vector_cosine_ops)
+    WITH (m = 16, ef_construction = 64);
+
+CREATE INDEX IF NOT EXISTS idx_chunks_document_id
+    ON chunks (document_id);
+
+
+-- ── Table 3: query_logs ──────────────────────────────────────────────────────
+-- Structured log of every query for evaluation and dashboarding.
+
+CREATE TABLE IF NOT EXISTS query_logs (
+    id                  BIGSERIAL PRIMARY KEY,
+    queried_at          TIMESTAMPTZ DEFAULT NOW(),
+
+    raw_query           TEXT,
+    rewritten_query     TEXT,
+    query_type          TEXT,
+
+    filters             JSONB,
+    retrieved_ark_ids   TEXT[],
+    response            TEXT,
+
+    relevancy_score     FLOAT,
+    faithfulness_score  FLOAT,
+    latency_ms          INT
+);
+"""
+
+
+def create_schema():
+    print("Creating schema ...")
+    with get_conn() as conn:
+        with get_cursor(conn) as cur:
+            cur.execute(DDL)
+    print("Done. Tables: documents, chunks, query_logs")
+
+
+if __name__ == "__main__":
+    create_schema()

embedding/embedder.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+from typing import List
+import numpy as np
+from config import BGE_MODEL_NAME, BGE_DEVICE, BGE_BATCH_SIZE
+from FlagEmbedding import BGEM3FlagModel
+
+
+class BGEEmbedder:
+
+    def __init__(self):
+        self._model = None
+
+    def _load(self):
+        if self._model is None:
+            print(f"Loading BGE-M3 model on {BGE_DEVICE}")
+            self._model = BGEM3FlagModel(
+                BGE_MODEL_NAME,
+                use_fp16=True,
+                device=BGE_DEVICE,
+            )
+
+    def encode_both(self, texts: List[str]) -> dict:
+        self._load()
+        texts = [t if t.strip() else " " for t in texts]
+        output = self._model.encode(
+            texts,
+            batch_size=BGE_BATCH_SIZE,
+            return_dense=True,
+            return_sparse=True,
+            return_colbert_vecs=False,
+        )
+        return {
+            "dense":  output["dense_vecs"].astype(np.float32),
+            "sparse": output["lexical_weights"],
+        }
+
+    def embed(self, texts: List[str]) -> np.ndarray:
+        return self.encode_both(texts)["dense"]
+
+    def embed_sparse(self, texts: List[str]) -> List[dict]:
+        return self.encode_both(texts)["sparse"]
+
+    def embed_one(self, text: str, is_query: bool = False) -> np.ndarray:
+        if is_query:
+            text = f"Represent this sentence for searching relevant passages: {text}"
+        return self.encode_both([text])["dense"][0]
+
+    def embed_one_sparse(self, text: str, is_query: bool = False) -> dict:
+        return {}
+
+    def encode_one_both(self, text: str, is_query: bool = False) -> dict:
+        if is_query:
+            text = f"Represent this sentence for searching relevant passages: {text}"
+        output = self.encode_both([text])
+        sparse_list = output["sparse"]
+        return {
+            "dense":  output["dense"][0],
+            "sparse": sparse_list[0] if sparse_list else {},
+        }
+
+    @staticmethod
+    def build_metadata_text(record: dict) -> str:
+        parts = []
+
+        if record.get("title"):
+            parts.append(f"Title: {record['title']}")
+
+        genres = record.get("genre") or []
+        if genres:
+            parts.append(f"Format: {', '.join(genres)}")
+
+        topics = record.get("topics") or []
+        if topics:
+            parts.append(f"Topics: {', '.join(topics)}")
+
+        geography = record.get("geography") or []
+        if geography:
+            parts.append(f"Geography: {', '.join(geography)}")
+
+        place = record.get("place") or []
+        if place:
+            parts.append(f"Place: {', '.join(place)}")
+
+        year = record.get("year") or []
+        if year:
+            parts.append(f"Year: {', '.join(str(y) for y in year)}")
+
+        collection = record.get("collection") or ""
+        if collection and collection != record.get("title"):
+            parts.append(f"Collection: {collection}")
+
+        # HTML already stripped at parse time, just truncate
+        abstract = record.get("abstract") or ""
+        if abstract:
+            parts.append(f"Description: {abstract}")
+
+        return " | ".join(parts)
+
+
+embedder = BGEEmbedder()

evaluation/eval.py

@@ -0,0 +1,232 @@
+"""
+evaluation/eval.py
+
+Runs the full pipeline against test_queries.jsonl and computes retrieval metrics.
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+import traceback
+from pathlib import Path
+from typing import List
+
+# ── Run from current_spring2026/ ──────────────────────────────────────────────
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from pipeline import run_query, PipelineResult
+
+
+# ── Metric helpers ────────────────────────────────────────────────────────────
+
+def hit_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> int:
+    return int(any(gt in retrieved[:k] for gt in ground_truths))
+
+
+def reciprocal_rank(retrieved: List[str], ground_truths: List[str]) -> float:
+    for i, ark_id in enumerate(retrieved, start=1):
+        if ark_id in ground_truths:
+            return 1.0 / i
+    return 0.0
+
+
+def recall_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> float:
+    if not ground_truths:
+        return 0.0
+    hits = sum(1 for gt in ground_truths if gt in retrieved[:k])
+    return hits / len(ground_truths)
+
+
+def precision_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> float:
+    if k == 0:
+        return 0.0
+    hits = sum(1 for ark in retrieved[:k] if ark in ground_truths)
+    return hits / k
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--queries",
+        default=str(Path(__file__).parent.parent / "test_queries.jsonl"),
+        help="Path to test_queries.jsonl",
+    )
+    parser.add_argument(
+        "--out",
+        default=str(Path(__file__).parent / "eval_results.csv"),
+        help="Path to save per-query CSV results",
+    )
+    args = parser.parse_args()
+
+    K_VALUES = [10, 30, 50]
+    MAX_K = max(K_VALUES)
+
+    queries_path = Path(args.queries)
+    if not queries_path.exists():
+        print(f"ERROR: test queries file not found at {queries_path}")
+        sys.exit(1)
+
+    with open(queries_path) as f:
+        entries = [json.loads(line) for line in f if line.strip()]
+
+    print(f"Loaded {len(entries)} queries from {queries_path}")
+    print(f"Evaluating top-{K_VALUES} retrieved results\n")
+
+    rows = []
+
+    for i, entry in enumerate(entries):
+        question    = entry["question"]
+        qtype       = entry["question_type"]
+        ground_truths = [
+            g["ark_id"].removeprefix("commonwealth:")
+            for g in entry.get("ground_truths", [])
+        ]
+        reference_answer = entry.get("answer", "")
+
+        print(f"[{i+1:02d}/{len(entries)}] ({qtype}) {question[:70]}...")
+
+        try:
+            result: PipelineResult = run_query(question, top_k=MAX_K)
+            retrieved_ids = [doc.ark_id for doc in result.documents]
+
+            mrr = reciprocal_rank(retrieved_ids, ground_truths)
+
+            # Hallucination test: pipeline should return no docs (or say "no results")
+            if qtype == "hallucination_test":
+                hallucination_pass = int(
+                    len(retrieved_ids) == 0
+                    or "no relevant" in result.generation.response.lower()
+                    or "not found" in result.generation.response.lower()
+                )
+            else:
+                hallucination_pass = ""
+
+            row = {
+                "question":          question,
+                "question_type":     qtype,
+                "classified_as":     result.intent.query_type,
+                "rewritten_query":   result.intent.rewritten_query,
+                "num_ground_truths": len(ground_truths),
+                "num_retrieved":     len(retrieved_ids),
+                "mrr":               round(mrr, 4),
+                "hallucination_pass": hallucination_pass,
+                "response_preview":  result.generation.response[:150].replace("\n", " "),
+                "retrieved_ids":     "|".join(retrieved_ids),
+                "ground_truth_ids":  "|".join(ground_truths),
+                "latency_ms":        result.latency_ms,
+                "error":             "",
+            }
+            for k in K_VALUES:
+                row[f"hit_at_{k}"]       = hit_at_k(retrieved_ids, ground_truths, k)
+                row[f"recall_at_{k}"]    = round(recall_at_k(retrieved_ids, ground_truths, k), 4)
+                row[f"precision_at_{k}"] = round(precision_at_k(retrieved_ids, ground_truths, k), 4)
+
+            status = "  " + "  ".join(f"hit@{k}={row[f'hit_at_{k}']}" for k in K_VALUES)
+            status += f"  mrr={mrr:.3f}"
+            if qtype == "hallucination_test":
+                status += f"  hallucination_pass={hallucination_pass}"
+            print(status)
+
+        except Exception as e:
+            traceback.print_exc()
+            print(f"  ERROR: {e}")
+            row = {
+                "question":          question,
+                "question_type":     qtype,
+                "classified_as":     "",
+                "rewritten_query":   "",
+                "num_ground_truths": len(ground_truths),
+                "num_retrieved":     0,
+                "mrr":               "",
+                "hallucination_pass": "",
+                "response_preview":  "",
+                "retrieved_ids":     "",
+                "ground_truth_ids":  "|".join(ground_truths),
+                "latency_ms":        "",
+                "error":             str(e),
+            }
+            for k in K_VALUES:
+                row[f"hit_at_{k}"]       = ""
+                row[f"recall_at_{k}"]    = ""
+                row[f"precision_at_{k}"] = ""
+
+        rows.append(row)
+
+    # ── Save CSV ──────────────────────────────────────────────────────────────
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = list(rows[0].keys())
+    with open(out_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(rows)
+    print(f"\nPer-query results saved to {out_path}")
+
+    # ── Summary by query type ─────────────────────────────────────────────────
+    print("\n" + "=" * 55)
+    print("SUMMARY")
+    print("=" * 55)
+
+    summary_rows = []
+
+    for qtype in ["metadata", "full_text", "hallucination_test"]:
+        subset = [r for r in rows if r["question_type"] == qtype and r["mrr"] != ""]
+        if not subset:
+            continue
+
+        n = len(subset)
+        avg = lambda key: sum(r[key] for r in subset) / n  # noqa: E731
+
+        print(f"\n{qtype}  (n={n})")
+        print(f"  MRR                  : {avg('mrr'):.3f}")
+
+        summary_row = {
+            "question_type": qtype,
+            "n": n,
+            "mrr": round(avg("mrr"), 4),
+            "hallucination_pass_rate": "",
+        }
+        for k in K_VALUES:
+            print(f"  Hit@{k:<2}               : {avg(f'hit_at_{k}'):.3f}")
+            print(f"  Recall@{k:<2}            : {avg(f'recall_at_{k}'):.3f}")
+            print(f"  Precision@{k:<2}         : {avg(f'precision_at_{k}'):.3f}")
+            summary_row[f"hit_at_{k}"]       = round(avg(f"hit_at_{k}"), 4)
+            summary_row[f"recall_at_{k}"]    = round(avg(f"recall_at_{k}"), 4)
+            summary_row[f"precision_at_{k}"] = round(avg(f"precision_at_{k}"), 4)
+
+        if qtype == "hallucination_test":
+            hall_subset = [r for r in rows if r["question_type"] == qtype and r["hallucination_pass"] != ""]
+            if hall_subset:
+                pass_rate = sum(r["hallucination_pass"] for r in hall_subset) / len(hall_subset)
+                print(f"  Hallucination pass   : {pass_rate:.3f}")
+                summary_row["hallucination_pass_rate"] = round(pass_rate, 4)
+
+        summary_rows.append(summary_row)
+
+    errors = [r for r in rows if r["error"]]
+    if errors:
+        print(f"\nFailed queries: {len(errors)}")
+        for r in errors:
+            print(f"  - {r['question'][:60]}: {r['error']}")
+
+    print()
+
+    # ── Save summary CSV ──────────────────────────────────────────────────────
+    if summary_rows:
+        summary_path = out_path.with_name(out_path.stem + "_summary.csv")
+        summary_fieldnames = list(summary_rows[0].keys())
+        with open(summary_path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=summary_fieldnames)
+            writer.writeheader()
+            writer.writerows(summary_rows)
+        print(f"Summary results saved to {summary_path}")
+
+
+if __name__ == "__main__":
+    main()

evaluation/eval_graphrag.py

@@ -0,0 +1,283 @@
+"""
+evaluation/eval_graphrag.py
+
+Evaluates two isolated retrieval systems on ALL queries:
+  System 1 — Dense + Sparse only
+  System 2 — GraphRAG only
+
+No filtering — every query is evaluated against both systems.
+Summary broken down by use_graph=True vs use_graph=False.
+
+Metrics: Hit@K, Recall@K, Precision@K for K = 5, 10, 20, 30, 50
+
+Run:
+    python -m evaluation.eval_graphrag
+    python -m evaluation.eval_graphrag --queries test_queries.jsonl --out evaluation/graphrag_results.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import List
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from retrieval.query_understanding import classify_query
+from retrieval.retriever import retrieve
+from graph.graph_retriever import retrieve_by_query
+from embedding.embedder import embedder
+from config import GRAPH_TOP_K
+
+
+
+K_VALUES = [5, 10, 20, 30, 50]
+MAX_K    = max(K_VALUES)
+
+
+# ── Metric helpers ────────────────────────────────────────────────────────────
+
+def hit_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> int:
+    return int(any(gt in retrieved[:k] for gt in ground_truths))
+
+
+def recall_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> float:
+    if not ground_truths:
+        return 0.0
+    hits = sum(1 for gt in ground_truths if gt in retrieved[:k])
+    return hits / len(ground_truths)
+
+
+def precision_at_k(retrieved: List[str], ground_truths: List[str], k: int) -> float:
+    if k == 0:
+        return 0.0
+    hits = sum(1 for ark in retrieved[:k] if ark in ground_truths)
+    return hits / k
+
+
+def compute_metrics(retrieved: List[str], ground_truths: List[str]) -> dict:
+    metrics = {}
+    for k in K_VALUES:
+        metrics[f"hit_{k}"]       = hit_at_k(retrieved, ground_truths, k)
+        metrics[f"recall_{k}"]    = round(recall_at_k(retrieved, ground_truths, k), 4)
+        metrics[f"precision_{k}"] = round(precision_at_k(retrieved, ground_truths, k), 4)
+    return metrics
+
+
+# ── GraphRAG-only retrieval ───────────────────────────────────────────────────
+
+def retrieve_graph_only(query_emb: np.ndarray, top_k: int = MAX_K) -> List[str]:
+    """Run GraphRAG retrieval only — no dense or sparse search."""
+    graph_results = retrieve_by_query(
+        query_embedding = query_emb,
+        exclude_ark_ids = set(),
+        top_k           = top_k,
+    )
+    return [r.ark_id for r in graph_results] if graph_results else []
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="GraphRAG vs Dense+Sparse evaluation")
+    parser.add_argument(
+        "--queries",
+        default=str(Path(__file__).parent.parent / "test_queries.jsonl"),
+    )
+    parser.add_argument(
+        "--out",
+        default=str(Path(__file__).parent / "graphrag_results.json"),
+    )
+    args = parser.parse_args()
+
+    queries_path = Path(args.queries)
+    if not queries_path.exists():
+        print(f"ERROR: test queries file not found at {queries_path}")
+        sys.exit(1)
+
+    with open(queries_path) as f:
+        all_entries = [json.loads(line) for line in f if line.strip()]
+
+    print(f"Loaded {len(all_entries)} queries\n")
+
+    # ── Classify all queries ──────────────────────────────────────────────────
+    print("Classifying all queries...")
+    classified = []
+    for entry in all_entries:
+        intent = classify_query(entry["question"])
+        classified.append((entry, intent))
+    print(f"  Classified {len(classified)} queries")
+    print(f"  use_graph=True  : {sum(1 for _, i in classified if i.use_graph)}")
+    print(f"  use_graph=False : {sum(1 for _, i in classified if not i.use_graph)}")
+    print(f"\nEvaluating at K = {K_VALUES}\n")
+
+    rows = []
+
+    for i, (entry, intent) in enumerate(classified):
+        question      = entry["question"]
+        qtype         = entry["question_type"]
+        ground_truths = [
+            g["ark_id"].removeprefix("commonwealth:")
+            for g in entry.get("ground_truths", [])
+        ]
+
+        print(f"[{i+1:02d}/{len(classified)}] {question[:70]}...")
+        print(f"  use_graph={intent.use_graph} | rewritten='{intent.rewritten_query[:60]}'")
+
+        try:
+            # ── System 1: Dense + Sparse ───────────────────────────────────
+            # retrieve() returns (documents, query_emb) — unpack both
+            dense_docs, query_emb = retrieve(intent, top_k=MAX_K)
+            dense_ids = [d.ark_id for d in dense_docs]
+
+            # ── System 2: GraphRAG only ────────────────────────────────────
+            # Reuse query_emb from dense retrieval — no re-embedding
+            graph_ids = retrieve_graph_only(query_emb, top_k=MAX_K)
+
+            # ── Compute metrics ────────────────────────────────────────────
+            dense_metrics = compute_metrics(dense_ids, ground_truths)
+            graph_metrics = compute_metrics(graph_ids, ground_truths)
+
+            # ── Build row ──────────────────────────────────────────────────
+            row = {
+                "question":          question,
+                "question_type":     qtype,
+                "rewritten_query":   intent.rewritten_query,
+                "use_graph":         intent.use_graph,
+                "num_ground_truths": len(ground_truths),
+                "dense_retrieved":   len(dense_ids),
+                "graph_retrieved":   len(graph_ids),
+                "ground_truth_ids":  ground_truths,
+                "dense_ids":         dense_ids,
+                "graph_ids":         graph_ids,
+                "error":             "",
+            }
+
+            for k in K_VALUES:
+                row[f"dense_hit_{k}"]       = dense_metrics[f"hit_{k}"]
+                row[f"dense_recall_{k}"]    = dense_metrics[f"recall_{k}"]
+                row[f"dense_precision_{k}"] = dense_metrics[f"precision_{k}"]
+                row[f"graph_hit_{k}"]       = graph_metrics[f"hit_{k}"]
+                row[f"graph_recall_{k}"]    = graph_metrics[f"recall_{k}"]
+                row[f"graph_precision_{k}"] = graph_metrics[f"precision_{k}"]
+                row[f"hit_delta_{k}"]       = graph_metrics[f"hit_{k}"]    - dense_metrics[f"hit_{k}"]
+                row[f"recall_delta_{k}"]    = round(graph_metrics[f"recall_{k}"]    - dense_metrics[f"recall_{k}"], 4)
+                row[f"precision_delta_{k}"] = round(graph_metrics[f"precision_{k}"] - dense_metrics[f"precision_{k}"], 4)
+
+            for k in [10, 30]:
+                d = dense_metrics[f"hit_{k}"]
+                g = graph_metrics[f"hit_{k}"]
+                print(f"  hit@{k}: dense={d}  graph={g}  Δ={g-d:+d}")
+
+        except Exception as e:
+            print(f"  ERROR: {e}")
+            row = {
+                "question":          question,
+                "question_type":     qtype,
+                "rewritten_query":   intent.rewritten_query,
+                "use_graph":         intent.use_graph,
+                "num_ground_truths": len(ground_truths),
+                "dense_retrieved":   0,
+                "graph_retrieved":   0,
+                "ground_truth_ids":  ground_truths,
+                "dense_ids":         [],
+                "graph_ids":         [],
+                "error":             str(e),
+            }
+            for k in K_VALUES:
+                for prefix in ["dense", "graph"]:
+                    row[f"{prefix}_hit_{k}"]       = ""
+                    row[f"{prefix}_recall_{k}"]    = ""
+                    row[f"{prefix}_precision_{k}"] = ""
+                row[f"hit_delta_{k}"]       = ""
+                row[f"recall_delta_{k}"]    = ""
+                row[f"precision_delta_{k}"] = ""
+
+        rows.append(row)
+
+    # ── Save JSON ─────────────────────────────────────────────────────────────
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    valid  = [r for r in rows if r["error"] == ""]
+    errors = [r for r in rows if r["error"] != ""]
+
+    def avg(key, subset):
+        vals = [r[key] for r in subset if r[key] != ""]
+        return round(sum(vals) / len(vals), 4) if vals else 0.0
+
+    def summary_for(subset):
+        if not subset:
+            return {}
+        return {
+            f"hit@{k}":       {"dense": avg(f"dense_hit_{k}", subset),       "graph": avg(f"graph_hit_{k}", subset),       "delta": round(avg(f"graph_hit_{k}", subset)       - avg(f"dense_hit_{k}", subset), 4)}
+            for k in K_VALUES
+        } | {
+            f"recall@{k}":    {"dense": avg(f"dense_recall_{k}", subset),    "graph": avg(f"graph_recall_{k}", subset),    "delta": round(avg(f"graph_recall_{k}", subset)    - avg(f"dense_recall_{k}", subset), 4)}
+            for k in K_VALUES
+        } | {
+            f"precision@{k}": {"dense": avg(f"dense_precision_{k}", subset), "graph": avg(f"graph_precision_{k}", subset), "delta": round(avg(f"graph_precision_{k}", subset) - avg(f"dense_precision_{k}", subset), 4)}
+            for k in K_VALUES
+        }
+
+    graph_queries    = [r for r in valid if r["use_graph"]]
+    no_graph_queries = [r for r in valid if not r["use_graph"]]
+
+    output = {
+        "metadata": {
+            "total_queries":   len(rows),
+            "valid_queries":   len(valid),
+            "failed_queries":  len(errors),
+            "use_graph_true":  len(graph_queries),
+            "use_graph_false": len(no_graph_queries),
+            "k_values":        K_VALUES,
+        },
+        "summary": {
+            "all_queries":     summary_for(valid),
+            "use_graph_true":  summary_for(graph_queries),
+            "use_graph_false": summary_for(no_graph_queries),
+        },
+        "results": rows,
+    }
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(output, f, indent=2)
+    print(f"\nResults saved to {out_path}")
+
+    # ── Print summary ─────────────────────────────────────────────────────────
+    for label, subset in [
+        ("ALL QUERIES",     valid),
+        ("use_graph=True",  graph_queries),
+        ("use_graph=False", no_graph_queries),
+    ]:
+        n_sub = len(subset)
+        if n_sub == 0:
+            continue
+        print(f"\n{'='*70}")
+        print(f"{label}  (n={n_sub})")
+        print(f"{'='*70}")
+        print(f"\n{'K':<6} {'Dense Hit':>10} {'Graph Hit':>10} {'Δ Hit':>8} {'Dense Rec':>10} {'Graph Rec':>10} {'Δ Rec':>8}")
+        print("-" * 70)
+        for k in K_VALUES:
+            dh = avg(f"dense_hit_{k}", subset)
+            gh = avg(f"graph_hit_{k}", subset)
+            dr = avg(f"dense_recall_{k}", subset)
+            gr = avg(f"graph_recall_{k}", subset)
+            print(
+                f"{k:<6} {dh:>10.3f} {gh:>10.3f} {gh-dh:>+8.3f} "
+                f"{dr:>10.3f} {gr:>10.3f} {gr-dr:>+8.3f}"
+            )
+
+    if errors:
+        print(f"\nFailed queries: {len(errors)}")
+        for r in errors:
+            print(f"  - {r['question'][:60]}: {r['error']}")
+    print()
+
+
+if __name__ == "__main__":
+    main()

evaluation/graphrag_results_spacy.json

@@ -0,0 +1,3226 @@
+{
+  "metadata": {
+    "total_queries": 51,
+    "valid_queries": 0,
+    "failed_queries": 51,
+    "use_graph_true": 0,
+    "use_graph_false": 0,
+    "k_values": [
+      5,
+      10,
+      20,
+      30,
+      50
+    ]
+  },
+  "summary": {
+    "all_queries": {},
+    "use_graph_true": {},
+    "use_graph_false": {}
+  },
+  "results": [
+    {
+      "question": "I want to see a variety of depictions of indigenous Americans from across the United States.",
+      "question_type": "metadata",
+      "rewritten_query": "depictions of indigenous Americans United States",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "70796j511",
+        "8g84ms67v",
+        "70796j910",
+        "1v53kk250"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find pictures of JFK before he became president.",
+      "question_type": "metadata",
+      "rewritten_query": "John F. Kennedy photographs before presidency",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "7w62g495q",
+        "7w62gc97q",
+        "7w62g5435",
+        "7w62gb97h"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "I have to do a project about the battle of Bunker Hill during the American Revolution, can you help me find some information about that?",
+      "question_type": "metadata",
+      "rewritten_query": "Battle of Bunker Hill American Revolution",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "3f462x95q",
+        "wd376807m",
+        "nc587w492",
+        "8c97n816r"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What were some important historical events that happened in Boston in 1919?",
+      "question_type": "metadata",
+      "rewritten_query": "important historical events in Boston 1919",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "2j62s563b",
+        "5h73vx18p",
+        "5h73qh68q",
+        "x920px83j"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What caused the Great Fire of Boston in 1872?",
+      "question_type": "metadata",
+      "rewritten_query": "causes of the Great Fire of Boston 1872",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "xp68kr375",
+        "xp68kr35m",
+        "xp68kr53j",
+        "c821gq65v"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find postcards of Cape Cod from the early 1900s",
+      "question_type": "metadata",
+      "rewritten_query": "postcards of Cape Cod",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "0k225t71f",
+        "cn69mp144",
+        "cn69mp02b",
+        "0k225t45t"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Show me depictions of women working in Boston during the early 1900s.",
+      "question_type": "hallucination_test",
+      "rewritten_query": "depictions of women working in Boston",
+      "use_graph": false,
+      "num_ground_truths": 0,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find materials related to abolitionist movements in Massachusetts.",
+      "question_type": "metadata",
+      "rewritten_query": "abolitionist movements in Massachusetts",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "m900pj71z",
+        "5h740p602",
+        "m900qg763",
+        "m900qr11x"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find old photographs of Beacon Hill, Boston.",
+      "question_type": "metadata",
+      "rewritten_query": "photographs of Beacon Hill, Boston",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "8623j613c",
+        "7h14d6210",
+        "ht24xf08g",
+        "9019x004d"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What was the Boston Tea Party and why did it happen?",
+      "question_type": "metadata",
+      "rewritten_query": "Boston Tea Party causes and events",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "d504v765w",
+        "zs2626294",
+        "f1884c31d",
+        "6d571125r"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What events took place in Boston during the American Revolution?",
+      "question_type": "metadata",
+      "rewritten_query": "events in Boston during the American Revolution",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "9s161871m",
+        "st74cw36c",
+        "wd3768027",
+        "dv142f145"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who was Crispus Attucks and what was his role in American history?",
+      "question_type": "metadata",
+      "rewritten_query": "Crispus Attucks role in American history",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "2801pm92n",
+        "vh53x3780",
+        "jq08ds22z",
+        "41688262q"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Are there any newspaper articles covering the women's suffrage movement in Massachusetts?",
+      "question_type": "full_text",
+      "rewritten_query": "women's suffrage movement in Massachusetts",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "mw232h306",
+        "4t64q022m",
+        "5d86wc88j",
+        "nv93cf51s"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find documents or letters from the Civil War era written by Massachusetts soldiers.",
+      "question_type": "hallucination_test",
+      "rewritten_query": "documents or letters from Civil War era Massachusetts soldiers",
+      "use_graph": false,
+      "num_ground_truths": 0,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What role did Frederick Douglass play in Massachusetts history?",
+      "question_type": "metadata",
+      "rewritten_query": "Frederick Douglass role in Massachusetts history",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "dv143022b",
+        "7s75fx33m",
+        "dv142z173",
+        "5m60th82r"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What did the Boston Public Library look like when it was first built?",
+      "question_type": "metadata",
+      "rewritten_query": "Boston Public Library original construction appearance",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "c821gx37x",
+        "gt54pc92n",
+        "7p88d826d",
+        "c821gx27p"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "When was the Longfellow Bridge between Boston and Cambridge built?",
+      "question_type": "metadata",
+      "rewritten_query": "Longfellow Bridge construction date",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "3f463c80z",
+        "8s45qv70f",
+        "pr76f347h",
+        "4f16kv932"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What were the biggest insurance companies in Boston in the early 20th century?",
+      "question_type": "metadata",
+      "rewritten_query": "biggest insurance companies in Boston early 20th century",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "9306wq12z",
+        "70797z85h",
+        "8k71nw59n",
+        "xp68kn172"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who was Samuel Adams and what did he do in Boston?",
+      "question_type": "metadata",
+      "rewritten_query": "Samuel Adams activities in Boston",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "fj2373328",
+        "jm217635p",
+        "2v23zv79k",
+        "7p88gj11s"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What was the role of the Boston Navy Yard?",
+      "question_type": "metadata",
+      "rewritten_query": "role of the Boston Navy Yard",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "2v240047f",
+        "1j92kc44m",
+        "8k71p1131",
+        "5425rv86w"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What was the Old State House in Boston used for?",
+      "question_type": "metadata",
+      "rewritten_query": "Old State House Boston usage",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "ht24xf50g",
+        "c534jd829",
+        "ht24xf521",
+        "5h73rs02f"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who was Paul Revere and what was his significance?",
+      "question_type": "metadata",
+      "rewritten_query": "Paul Revere significance",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "wh246s53h",
+        "cr56qq138",
+        "fj2372144",
+        "gt54mc309"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What happened at the Boston Massacre?",
+      "question_type": "metadata",
+      "rewritten_query": "Boston Massacre events",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "jq08ds22z",
+        "vh53x374w",
+        "vh53x3780",
+        "dv142f145"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who were the Sons of Liberty?",
+      "question_type": "metadata",
+      "rewritten_query": "Sons of Liberty",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "fj2372543",
+        "vd66w6545",
+        "9880w844w",
+        "zs2626294"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What is Faneuil Hall's historical significance?",
+      "question_type": "metadata",
+      "rewritten_query": "Faneuil Hall historical significance",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "p2677g44k",
+        "7m01kw73f",
+        "v405sz97p",
+        "xp68kq91r"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What was the Big Dig and why was it significant?",
+      "question_type": "metadata",
+      "rewritten_query": "Big Dig significance",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "t722qd95f",
+        "ww72m211q",
+        "4q77fs75b",
+        "cn69pp161"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What was the history of the Boston Common?",
+      "question_type": "metadata",
+      "rewritten_query": "history of the Boston Common",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "c821gs31f",
+        "2801pn560",
+        "2801pn446",
+        "9g54z626j"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Why is Boston named Boston?",
+      "question_type": "metadata",
+      "rewritten_query": "origin of the name Boston",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "z316sk199",
+        "m900r167m",
+        "z316sk156",
+        "8k71p274z"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Show me some newspapers covering the 1916 World Series.",
+      "question_type": "full_text",
+      "rewritten_query": "1916 World Series newspapers",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "x346mh47g",
+        "r7823s29k",
+        "c247n520p",
+        "g158jw90g"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Give me some historical info on the Zulu people in South Africa.",
+      "question_type": "metadata",
+      "rewritten_query": "historical information on the Zulu people in South Africa",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "1z40mh38h",
+        "br86fd210",
+        "jm218041r",
+        "7p88cv41v"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Show me poems by Emily Dickinson, in her own handwriting",
+      "question_type": "metadata",
+      "rewritten_query": "poems by Emily Dickinson in her handwriting",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "fq977x21b",
+        "kh04mw646",
+        "fq977x56f",
+        "fq977z101"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who was William Lloyd Garrison and what did he do?",
+      "question_type": "metadata",
+      "rewritten_query": "William Lloyd Garrison biography and achievements",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "2514p017p",
+        "8c97n842c",
+        "70796c89x",
+        "2v23x8010"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Were there ever any breweries based in Boston?",
+      "question_type": "metadata",
+      "rewritten_query": "breweries based in Boston",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "br86bd53r",
+        "t435gm15v",
+        "t435gm11r",
+        "t435gm032"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What were the results of the 1935 Boston Marathon?",
+      "question_type": "full_text",
+      "rewritten_query": "1935 Boston Marathon results",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "3t946b82t",
+        "3t946c14w",
+        "1j92pv18d",
+        "3t946b36q"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Can you help me find some old dessert recipes from the 1800s?",
+      "question_type": "full_text",
+      "rewritten_query": "dessert recipes from the 1800s",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "j0992n65t",
+        "j0992q789",
+        "xk81rw520",
+        "w663bd023"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "I am writing a report about the history of people making boats in Massachusetts, can you help me find some information?",
+      "question_type": "metadata",
+      "rewritten_query": "history of boat making in Massachusetts",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "cn69mf744",
+        "1831dm900",
+        "n009xf24t",
+        "3r075h422"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "I want to find some examples of what tickets to a baseball game looked like back in the old days.",
+      "question_type": "metadata",
+      "rewritten_query": "baseball game tickets examples",
+      "use_graph": false,
+      "num_ground_truths": 3,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "gq67jx919",
+        "gq67jx97z",
+        "gq67jz002"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Who were the Boston Americans?",
+      "question_type": "metadata",
+      "rewritten_query": "Boston Americans baseball team",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "sf268644d",
+        "sf268560k",
+        "sf268672k",
+        "sf268850z"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Can you help me find some historical maps of Boston's subway system from the early 1900s?",
+      "question_type": "metadata",
+      "rewritten_query": "historical maps of Boston subway system",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "4x51m294f",
+        "0v83bg708",
+        "g732gs78m",
+        "34852076k"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "My class is reading Moby Dick and I have to do a project about whaling in Massachusetts, can you help me find some sources?",
+      "question_type": "metadata",
+      "rewritten_query": "whaling in Massachusetts",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "rv049z346",
+        "5h73rx028",
+        "cv43p6866",
+        "nk322p393"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Is it true that Harvard used to have a good football team?",
+      "question_type": "full_text",
+      "rewritten_query": "Harvard football team historical performance",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "90200b57q",
+        "g158jv887",
+        "t148nn16h",
+        "0z70h6684"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find some newspaper articles about the sinking of the Lusitania during WWI",
+      "question_type": "full_text",
+      "rewritten_query": "sinking of the Lusitania during World War I",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "0v83gc74f",
+        "v118zs65j",
+        "7d27h447j",
+        "4b29jq26h"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Were there any people from Boston on the Titanic when it sank?",
+      "question_type": "full_text",
+      "rewritten_query": "Boston passengers on the Titanic",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "sn00j357m",
+        "nc586t329",
+        "xd07q011v",
+        "5138rj364"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What caused the Great Depression?",
+      "question_type": "full_text",
+      "rewritten_query": "causes of the Great Depression",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "sn00jh80w",
+        "jq08d420g",
+        "8k71w2448",
+        "kw52rs605"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "What information can you find about the Coconut Grove fire disaster?",
+      "question_type": "metadata",
+      "rewritten_query": "Coconut Grove fire disaster",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "x346fh915",
+        "x346fh99c",
+        "7s75dh42x",
+        "7s75dh42x"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "How did Boston mark the end of Prohibition?",
+      "question_type": "full_text",
+      "rewritten_query": "Boston end of Prohibition celebrations",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "h128w283j",
+        "g732mx538",
+        "q524n563d",
+        "zc781f69b"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find some information about Charles Ponzi, including newspaper articles and photographs if possible",
+      "question_type": "full_text",
+      "rewritten_query": "Charles Ponzi",
+      "use_graph": true,
+      "num_ground_truths": 5,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "6682zs44c",
+        "b8515n701",
+        "sb39fz533",
+        "xs55tr436",
+        "6t059v54k"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "I am writing a report about historical presidential assassinations, can you help me find some sources?",
+      "question_type": "full_text",
+      "rewritten_query": "historical presidential assassinations",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "5x21v606k",
+        "cn69tz48n",
+        "3b597p07t",
+        "bz60m1252"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find some information about the Bread and Roses strike in Lawrence, Mass., including pictures",
+      "question_type": "full_text",
+      "rewritten_query": "Bread and Roses strike Lawrence Massachusetts",
+      "use_graph": true,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "wd3794346",
+        "2801ss217",
+        "dz016w71v",
+        "1n79q7966"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find a variety of newspaper articles about women who were accused of killing their husbands",
+      "question_type": "full_text",
+      "rewritten_query": "newspaper articles about women accused of killing their husbands",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "7079cb86r",
+        "1v53sm39k",
+        "m326t676k",
+        "x920pd34h"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    },
+    {
+      "question": "Find some newspaper articles about major snowstorms that have hit Massachusetts over the years",
+      "question_type": "full_text",
+      "rewritten_query": "major snowstorms in Massachusetts",
+      "use_graph": false,
+      "num_ground_truths": 4,
+      "dense_retrieved": 0,
+      "graph_retrieved": 0,
+      "ground_truth_ids": [
+        "0z70h781w",
+        "pg15jt06t",
+        "v4060z71q",
+        "d7920t25x"
+      ],
+      "dense_ids": [],
+      "graph_ids": [],
+      "error": "list index out of range",
+      "dense_hit_5": "",
+      "dense_recall_5": "",
+      "dense_precision_5": "",
+      "graph_hit_5": "",
+      "graph_recall_5": "",
+      "graph_precision_5": "",
+      "hit_delta_5": "",
+      "recall_delta_5": "",
+      "precision_delta_5": "",
+      "dense_hit_10": "",
+      "dense_recall_10": "",
+      "dense_precision_10": "",
+      "graph_hit_10": "",
+      "graph_recall_10": "",
+      "graph_precision_10": "",
+      "hit_delta_10": "",
+      "recall_delta_10": "",
+      "precision_delta_10": "",
+      "dense_hit_20": "",
+      "dense_recall_20": "",
+      "dense_precision_20": "",
+      "graph_hit_20": "",
+      "graph_recall_20": "",
+      "graph_precision_20": "",
+      "hit_delta_20": "",
+      "recall_delta_20": "",
+      "precision_delta_20": "",
+      "dense_hit_30": "",
+      "dense_recall_30": "",
+      "dense_precision_30": "",
+      "graph_hit_30": "",
+      "graph_recall_30": "",
+      "graph_precision_30": "",
+      "hit_delta_30": "",
+      "recall_delta_30": "",
+      "precision_delta_30": "",
+      "dense_hit_50": "",
+      "dense_recall_50": "",
+      "dense_precision_50": "",
+      "graph_hit_50": "",
+      "graph_recall_50": "",
+      "graph_precision_50": "",
+      "hit_delta_50": "",
+      "recall_delta_50": "",
+      "precision_delta_50": ""
+    }
+  ]
+}

evaluation/logger.py

@@ -0,0 +1,126 @@
+"""
+evaluation/logger.py
+
+Structured logger for every query that passes through the pipeline.
+Writes to:
+  1. The `query_logs` table in PostgreSQL (for dashboarding and SQL analysis)
+  2. A local CSV file (for portability and DeepEval input)
+
+Usage:
+    from evaluation.logger import log_query
+    log_query(intent, retrieved_docs, generation_result, latency_ms=240)
+"""
+
+from __future__ import annotations
+
+import csv
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional
+
+from database.schema import get_conn, get_cursor
+from retrieval.query_understanding import QueryIntent
+from retrieval.retriever import RetrievedDocument
+from generation.generator import GenerationResult
+
+LOG_CSV_PATH = Path("logs/query_log.csv")
+
+CSV_HEADERS = [
+    "queried_at",
+    "raw_query",
+    "rewritten_query",
+    "query_type",
+    "year_min",
+    "year_max",
+    "geography",
+    "topics",
+    "retrieved_ark_ids",
+    "response",
+    "latency_ms",
+    "relevancy_score",
+    "faithfulness_score",
+]
+
+
+def _ensure_csv(path: Path):
+    """Create CSV with headers if it doesn't exist."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not path.exists():
+        with open(path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=CSV_HEADERS)
+            writer.writeheader()
+
+
+def log_query(
+    intent:            QueryIntent,
+    retrieved_docs:    List[RetrievedDocument],
+    generation_result: GenerationResult,
+    latency_ms:        int = 0,
+    relevancy_score:   Optional[float] = None,
+    faithfulness_score: Optional[float] = None,
+):
+    """
+    Persist one query event to the DB and CSV log.
+    Fails silently on DB errors so a logging failure never breaks the search UX.
+    """
+    queried_at      = datetime.now(timezone.utc).isoformat()
+    retrieved_arks  = [d.ark_id for d in retrieved_docs]
+    filters_json    = json.dumps({
+        "year_min":  intent.date_filter.year_min,
+        "year_max":  intent.date_filter.year_max,
+        "doc_types": intent.doc_types,
+    })
+
+    # ── 1. Write to PostgreSQL ──────────────────────────────────────────────
+    try:
+        with get_conn() as conn:
+            with get_cursor(conn) as cur:
+                cur.execute(
+                    """
+                    INSERT INTO query_logs (
+                        queried_at, raw_query, rewritten_query, query_type,
+                        filters, retrieved_ark_ids, response,
+                        relevancy_score, faithfulness_score, latency_ms
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                    """,
+                    (
+                        queried_at,
+                        intent.raw_query,
+                        intent.rewritten_query,
+                        intent.query_type,
+                        filters_json,
+                        retrieved_arks,
+                        generation_result.response,
+                        relevancy_score,
+                        faithfulness_score,
+                        latency_ms,
+                    ),
+                )
+    except Exception as e:
+        print(f"[logger] DB write failed (non-fatal): {e}")
+
+    # ── 2. Append to CSV ────────────────────────────────────────────────────
+    try:
+        _ensure_csv(LOG_CSV_PATH)
+        row = {
+            "queried_at":        queried_at,
+            "raw_query":         intent.raw_query,
+            "rewritten_query":   intent.rewritten_query,
+            "query_type":        intent.query_type,
+            "year_min":          intent.date_filter.year_min,
+            "year_max":          intent.date_filter.year_max,
+            "geography":         "",
+            "topics":            "",
+            "retrieved_ark_ids": "|".join(retrieved_arks),
+            "response":          generation_result.response,
+            "latency_ms":        latency_ms,
+            "relevancy_score":   relevancy_score,
+            "faithfulness_score": faithfulness_score,
+        }
+        with open(LOG_CSV_PATH, "a", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=CSV_HEADERS)
+            writer.writerow(row)
+    except Exception as e:
+        print(f"[logger] CSV write failed (non-fatal): {e}")

generation/generator.py

@@ -0,0 +1,232 @@
+# """
+# generation/generator.py
+
+# Passes the top retrieved documents + their best chunk text to GPT-4o
+# and generates a contextually grounded response.
+
+# Design constraints (per client feedback):
+#   - Response must be anchored to retrieved content — no hallucination
+#   - Not a chatbot — a structured search result with explanation
+#   - Should explain why results are relevant
+# """
+
+# from __future__ import annotations
+
+# from dataclasses import dataclass
+# from typing import List
+
+# from openai import OpenAI
+
+# from config import OPENAI_API_KEY, OPENAI_CHAT_MODEL, MAX_CONTEXT_CHUNKS, GENERATION_MAX_TOKENS
+# from retrieval.retriever import RetrievedDocument
+
+# client = OpenAI(api_key=OPENAI_API_KEY)
+
+
+# # ── Result dataclass ──────────────────────────────────────────────────────────
+
+# @dataclass
+# class GenerationResult:
+#     response:      str
+#     source_titles: List[str]
+#     source_urls:   List[str]
+
+
+# # ── Prompt builders ───────────────────────────────────────────────────────────
+
+# SYSTEM_PROMPT = """
+# You are a search assistant for the Boston Public Library's Digital Commonwealth archive,
+# which contains historical newspapers, photographs, maps, manuscripts, and other materials
+# from Massachusetts institutions.
+
+# You will be given:
+# 1. A user's search query
+# 2. A set of retrieved documents with their metadata and a relevant excerpt
+
+# Your task:
+# - Write a 2-4 sentence explanation of what was found and why these materials are relevant
+# - Ground your explanation ONLY in the retrieved documents provided
+# - Do not invent facts, dates, or events not present in the excerpts
+# - Use a clear, accessible tone suitable for researchers, students, and the general public
+# - Do not mention scores, rankings, or technical retrieval details
+# - If the results are from a narrow time period or place, mention that context
+# """.strip()
+
+
+# def _build_context(docs: List[RetrievedDocument], max_chunks: int = MAX_CONTEXT_CHUNKS) -> str:
+#     """
+#     Format the top retrieved documents as a context block for GPT-4o.
+#     Only includes documents that have a non-empty chunk text.
+#     """
+#     lines = []
+#     included = 0
+
+#     for i, doc in enumerate(docs):
+#         if not doc.best_chunk_text.strip():
+#             continue
+#         if included >= max_chunks:
+#             break
+
+#         lines.append(f"[Document {included + 1}]")
+#         lines.append(f"Title     : {doc.title}")
+#         lines.append(f"Date      : {doc.issue_date or ', '.join(str(y) for y in doc.year)}")
+#         lines.append(f"Institution: {doc.institution}")
+#         if doc.topics:
+#             lines.append(f"Topics    : {', '.join(doc.topics)}")
+#         if doc.geography:
+#             lines.append(f"Geography : {', '.join(doc.geography)}")
+#         lines.append(f"Excerpt   : {doc.best_chunk_text[:600]}")   # cap excerpt length
+#         lines.append("")
+
+#         included += 1
+
+#     return "\n".join(lines)
+
+
+# # ── Generator ─────────────────────────────────────────────────────────────────
+
+# def generate(raw_query: str, docs: List[RetrievedDocument]) -> GenerationResult:
+#     """
+#     Generate a response given the user query and retrieved documents.
+
+#     Returns a GenerationResult with the response text and source metadata.
+#     """
+#     if not docs:
+#         return GenerationResult(
+#             response      = "No relevant materials were found for your query. "
+#                             "Try rephrasing or broadening your search.",
+#             source_titles = [],
+#             source_urls   = [],
+#         )
+
+#     context = _build_context(docs)
+
+#     user_message = f"""User query: {raw_query}
+
+# Retrieved documents:
+# {context}
+
+# Please write a brief explanation of what was found and why these materials are relevant to the query.
+# """
+
+#     response = client.chat.completions.create(
+#         model       = OPENAI_CHAT_MODEL,
+#         temperature = 0.3,      # slight creativity for natural prose, still grounded
+#         max_tokens  = GENERATION_MAX_TOKENS,
+#         messages    = [
+#             {"role": "system", "content": SYSTEM_PROMPT},
+#             {"role": "user",   "content": user_message},
+#         ],
+#     )
+
+#     response_text = response.choices[0].message.content.strip()
+
+#     return GenerationResult(
+#         response      = response_text,
+#         source_titles = [d.title     for d in docs],
+#         source_urls   = [d.source_url for d in docs],
+#     )
+
+"""
+generation/generator.py
+
+Generates a single grounded summary that cites each retrieved document
+inline by number, e.g. [1], [2], so users can trace claims to results.
+"""
+
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import List
+from openai import OpenAI
+
+from config import OPENAI_API_KEY, OPENAI_CHAT_MODEL, GENERATION_MAX_TOKENS
+from retrieval.retriever import RetrievedDocument
+
+client = OpenAI(api_key=OPENAI_API_KEY)
+
+
+@dataclass
+class GenerationResult:
+    response:      str
+    source_titles: List[str]
+    source_urls:   List[str]
+
+
+SYSTEM_PROMPT = """
+You are a search assistant for the Boston Public Library's Digital Commonwealth archive,
+which contains historical newspapers, photographs, maps, manuscripts, and other materials
+from Massachusetts institutions.
+
+You will be given a user query and a numbered list of retrieved documents with their
+metadata and text excerpts.
+
+Write a 3-5 sentence response that:
+- Summarizes what was found and why it is relevant to the query
+- Cites specific documents inline using their number, e.g. [1], [2], [3]
+- Is grounded ONLY in the provided documents — do not invent facts
+- Uses clear, accessible language suitable for researchers and the general public
+- Does not mention scores, rankings, or technical retrieval details
+
+Example format:
+"Several materials related to the 1919 Boston Molasses Disaster are available [1][2].
+The Boston Traveler covered the event extensively in its January 1919 issues [1],
+while photographs of the aftermath document the structural damage to the North End [3]."
+
+If the documents are not relevant to the query, say so clearly and suggest refining the search.
+""".strip()
+
+
+def _build_context(docs: List[RetrievedDocument]) -> str:
+    """Format retrieved documents as a numbered list for GPT-4o."""
+    lines = []
+    for i, doc in enumerate(docs, start=1):
+        date_str = doc.issue_date or (str(doc.year[0]) if doc.year else "unknown date")
+        excerpt  = doc.best_chunk_text[:300] if doc.best_chunk_text else "No text excerpt — collection-level record."
+        lines.append(f"[{i}] Title: {doc.title}")
+        lines.append(f"    Date: {date_str} | Institution: {doc.institution}")
+        if doc.topics:
+            lines.append(f"    Topics: {', '.join(doc.topics)}")
+        lines.append(f"    Excerpt: {excerpt}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def generate(raw_query: str, docs: List[RetrievedDocument]) -> GenerationResult:
+    """
+    Generate a single cited summary referencing documents by [number].
+    """
+    if not docs:
+        return GenerationResult(
+            response      = "No relevant materials were found for your query. Try rephrasing or using a more specific historical topic.",
+            source_titles = [],
+            source_urls   = [],
+        )
+
+    context = _build_context(docs)
+
+    user_message = f"""User query: {raw_query}
+
+Retrieved documents:
+{context}
+
+Write a concise summary that cites the relevant documents inline by number."""
+
+    response = client.chat.completions.create(
+        model       = OPENAI_CHAT_MODEL,
+        temperature = 0.2,
+        max_tokens  = GENERATION_MAX_TOKENS,
+        messages    = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user",   "content": user_message},
+        ],
+    )
+
+    if not response.choices:
+        raise ValueError("OpenAI returned empty choices (finish_reason may indicate content filter)")
+    response_text = response.choices[0].message.content.strip()
+
+    return GenerationResult(
+        response      = response_text,
+        source_titles = [d.title for d in docs],
+        source_urls   = [d.source_url for d in docs],
+    )

ingestion/chunker.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+from typing import List
+from transformers import AutoTokenizer
+from config import CHUNK_SIZE, CHUNK_OVERLAP
+
+class Chunker:
+    def __init__(self, chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP):
+        self.chunk_size    = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.enc           = AutoTokenizer.from_pretrained("BAAI/bge-m3")
+
+    def chunk(self, text: str) -> List[str]:
+        if not text or not text.strip():
+            return []
+
+        tokens = self.enc.encode(text, add_special_tokens=False)
+
+        if len(tokens) == 0:
+            return []
+
+        chunks = []
+        start  = 0
+        step   = self.chunk_size - self.chunk_overlap
+
+        while start < len(tokens):
+            end        = min(start + self.chunk_size, len(tokens))
+            chunk_ids  = tokens[start:end]
+            chunk_text = self.enc.decode(chunk_ids, skip_special_tokens=True).strip()
+            if chunk_text:
+                chunks.append(chunk_text)
+            if end == len(tokens):
+                break
+            start += step
+
+        return chunks
+
+    def chunk_record(self, record: dict) -> List[dict]:
+        raw_text = record.get("clean_text") or record.get("raw_text") or ""
+        texts    = self.chunk(raw_text)
+        return [
+            {
+                "ark_id":      record["ark_id"],
+                "chunk_index": i,
+                "chunk_text":  text,
+            }
+            for i, text in enumerate(texts)
+        ]
+
+
+# Module-level singleton
+chunker = Chunker()

ingestion/ingest.py

@@ -0,0 +1,411 @@
+"""
+ingestion/ingest.py
+
+Ingestion pipeline for two data sources:
+
+Source A — Full-text records (PRIMARY)
+  Layout: data/fulltext/<collection_name>/<year>.json
+  Structure: { "year": 1946, "records": [ {record}, ... ] }
+  Fields: flat, includes clean_text (or raw_text) and all metadata
+  These get both metadata + chunk embeddings (dense + sparse).
+
+Source B — Metadata-only records (SECONDARY)
+  Layout: data/metadata/metadata.jsonl
+  Structure: { "data": { "id": "commonwealth:...", "attributes": { ... } } }
+  These get only metadata embeddings (dense + sparse). No chunks.
+
+Run:
+    python -m ingestion.ingest
+    python -m ingestion.ingest --fulltext-dir data/fulltext --skip-metadata
+    python -m ingestion.ingest --metadata-file data/metadata/metadata.jsonl --skip-fulltext
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator, List
+
+import numpy as np
+from psycopg2.extras import execute_values
+
+from config import MIN_CHAR_COUNT, BGE_BATCH_SIZE
+from database.schema import get_conn, get_cursor
+from embedding.embedder import embedder
+from ingestion.chunker import chunker
+import re
+import html
+
+DEFAULT_FULLTEXT_DIR  = "data/fulltext"
+DEFAULT_METADATA_FILE = "data/metadata/metadata.jsonl"
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+def sparse_to_arrays(sparse: dict) -> tuple[list, list]:
+    """Convert {token_id: weight} dict to (token_ids[], weights[]) arrays."""
+    token_ids = [int(k) for k in sparse.keys()]
+    weights   = [float(v) for v in sparse.values()]
+    return token_ids, weights
+
+
+def is_valid(record: dict) -> bool:
+    return bool(record.get("ark_id"))
+
+
+def has_fulltext(record: dict) -> bool:
+    text = record.get("clean_text") or record.get("raw_text") or ""
+    return (
+        not record.get("_metadata_only", False)
+        and len(text) >= MIN_CHAR_COUNT
+    )
+
+
+def parse_date_start(record: dict):
+    raw = record.get("date_start", "") or ""
+    if not raw:
+        return None
+    try:
+        return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+
+
+# ── Source A: Full-text records ───────────────────────────────────────────────
+
+def iter_fulltext_records(fulltext_dir: str) -> Iterator[dict]:
+    root = Path(fulltext_dir)
+    if not root.exists():
+        print(f"  [ingest] Full-text dir not found: {fulltext_dir} — skipping.")
+        return
+
+    json_files = sorted(root.rglob("*.json"))
+    if not json_files:
+        print(f"  [ingest] No .json files found under {fulltext_dir}")
+        return
+
+    for fpath in json_files:
+        collection = fpath.parent.name
+        print(f"  Reading {fpath.relative_to(root)} ...")
+        with open(fpath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        for rec in data.get("records", []):
+            if not rec.get("collection"):
+                rec["collection"] = collection
+            yield rec
+
+
+# ── Source B: Metadata-only records ──────────────────────────────────────────
+
+def _parse_metadata_record(raw: dict) -> dict:
+    data      = raw.get("data", {})
+    attrs     = data.get("attributes", {})
+    record_id = data.get("id", attrs.get("id", ""))
+    ark_id    = record_id.split(":")[-1] if ":" in record_id else record_id
+    abstract_raw = attrs.get("abstract_tsi", "") or ""
+    abstract_unescaped = html.unescape(abstract_raw)
+    abstract_clean = re.sub(r"<[^>]+>", " ", abstract_unescaped).strip()
+    abstract_clean = re.sub(r"\s+", " ", abstract_clean)
+    
+
+    return {
+        "ark_id":         ark_id,
+        "record_id":      record_id,
+        "source_url":     attrs.get("identifier_uri_ss", ""),
+        "iiif_manifest":  attrs.get("identifier_iiif_manifest_ss", ""),
+        "newspaper":      "",
+        "collection":     attrs.get("title_info_primary_tsi", ""),
+        "title":          attrs.get("title_info_primary_tsi", ""),
+        "issue_date":     attrs.get("title_info_partnum_tsi", ""),
+        "date_iso":       attrs.get("date_edtf_ssm", []),
+        "date_start":     attrs.get("date_start_dtsi", ""),
+        "year":           attrs.get("date_facet_yearly_itim", []),
+        "publisher":      attrs.get("publisher_tsim", []),
+        "place":          attrs.get("publication_place_tsim", []),
+        "language":       attrs.get("language_ssim", []),
+        "institution":    attrs.get("institution_name_ssi", ""),
+        "page_count":     len(attrs.get("filenames_ssim", [])),
+        "pages":          attrs.get("filenames_ssim", []),
+        "topics":         attrs.get("subject_topic_tsim", []),
+        "geography":      attrs.get("subject_geographic_ssim", []),
+        "clean_text":     "",
+        "raw_text":       "",
+        "char_count":     0,
+        "ingested_at":    datetime.now(timezone.utc).isoformat(),
+        "_metadata_only": True,
+        "genre":          attrs.get("genre_basic_ssim", []),
+        # Strip HTML once at parse time instead of repeatedly at embed time
+        "exemplary_image_id": attrs.get("exemplary_image_ssi", ""),
+        "abstract": abstract_clean,
+    }
+
+
+def iter_metadata_records(
+    metadata_file: str,
+    seen_record_ids: set,
+) -> Iterator[dict]:
+    fpath = Path(metadata_file)
+    if not fpath.exists():
+        print(f"  [ingest] Metadata file not found: {metadata_file} — skipping.")
+        return
+
+    print(f"  Reading metadata JSONL: {fpath.name} ...")
+    skipped = 0
+    with open(fpath, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                raw = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            rec = _parse_metadata_record(raw)
+            if rec["record_id"] in seen_record_ids:
+                skipped += 1
+                continue
+            yield rec
+
+    if skipped:
+        print(f"  Skipped {skipped} metadata records already covered by full-text source.")
+
+
+# ── DB writes ─────────────────────────────────────────────────────────────────
+
+def upsert_documents(
+    records: List[dict],
+    meta_embeddings: np.ndarray,
+    meta_sparse: List[dict],
+    conn,
+) -> dict:
+    """Upsert into documents table. Returns {ark_id: db_id}."""
+    rows = []
+    for rec, emb, sparse in zip(records, meta_embeddings, meta_sparse):
+        token_ids, weights = sparse_to_arrays(sparse)
+        rows.append((
+            rec["ark_id"],
+            rec.get("record_id", ""),
+            rec.get("source_url", ""),
+            rec.get("iiif_manifest", ""),
+            rec.get("newspaper", ""),
+            rec.get("collection", ""),
+            rec.get("institution", ""),
+            rec.get("title", ""),
+            rec.get("issue_date", ""),
+            rec.get("date_iso") or [],
+            parse_date_start(rec),
+            rec.get("year") or [],
+            rec.get("publisher") or [],
+            rec.get("place") or [],
+            rec.get("language") or [],
+            rec.get("page_count", 0),
+            rec.get("pages") or [],
+            rec.get("topics") or [],
+            rec.get("geography") or [],
+            rec.get("char_count", 0),
+            rec.get("genre") or [],
+            rec.get("abstract", ""),
+            rec.get("exemplary_image_id", ""),
+            emb.tolist(),
+            token_ids,
+            weights,
+            rec.get("ingested_at", datetime.now(timezone.utc).isoformat()),
+        ))
+
+    sql = """
+        INSERT INTO documents (
+            ark_id, record_id, source_url, iiif_manifest,
+            newspaper, collection, institution,
+            title, issue_date, date_iso, date_start, year,
+            publisher, place, language,
+            page_count, pages, topics, geography,
+            char_count,
+            genre, abstract, exemplary_image_id,
+            metadata_embedding, sparse_token_ids, sparse_weights,
+            ingested_at
+        )
+        VALUES %s
+        ON CONFLICT (ark_id) DO UPDATE SET
+            metadata_embedding = EXCLUDED.metadata_embedding,
+            sparse_token_ids   = EXCLUDED.sparse_token_ids,
+            sparse_weights     = EXCLUDED.sparse_weights,
+            char_count         = EXCLUDED.char_count,
+            genre               = EXCLUDED.genre,
+            abstract            = EXCLUDED.abstract,
+            exemplary_image_id  = EXCLUDED.exemplary_image_id,
+            ingested_at        = EXCLUDED.ingested_at
+        RETURNING ark_id, id
+    """
+    with get_cursor(conn) as cur:
+        results = execute_values(cur, sql, rows, fetch=True)
+    return {row["ark_id"]: row["id"] for row in results}
+
+
+def insert_chunks(
+    chunk_rows: List[dict],
+    text_embeddings: np.ndarray,
+    sparse_embeddings: List[dict],
+    ark_to_doc_id: dict,
+    conn,
+):
+    if not chunk_rows:
+        return
+
+    doc_ids = list({
+        ark_to_doc_id[c["ark_id"]]
+        for c in chunk_rows
+        if c["ark_id"] in ark_to_doc_id
+    })
+
+    with get_cursor(conn) as cur:
+        cur.execute("DELETE FROM chunks WHERE document_id = ANY(%s)", (doc_ids,))
+
+    rows = []
+    for chunk, emb, sparse in zip(chunk_rows, text_embeddings, sparse_embeddings):
+        doc_id = ark_to_doc_id.get(chunk["ark_id"])
+        if doc_id is None:
+            continue
+        token_ids, weights = sparse_to_arrays(sparse)
+        rows.append((
+            doc_id,
+            chunk["ark_id"],
+            chunk["chunk_index"],
+            chunk["chunk_text"],
+            emb.tolist(),
+            token_ids,
+            weights,
+        ))
+
+    with get_cursor(conn) as cur:
+        execute_values(
+            cur,
+            """
+            INSERT INTO chunks (
+                document_id, ark_id, chunk_index, chunk_text,
+                text_embedding, sparse_token_ids, sparse_weights
+            )
+            VALUES %s
+            """,
+            rows,
+        )
+
+
+# ── Batch flush ───────────────────────────────────────────────────────────────
+
+def flush_batch(batch: List[dict]) -> tuple[int, int]:
+    """Embed and write one batch. Returns (n_docs, n_chunks)."""
+
+    # Single forward pass for metadata — dense + sparse together
+    meta_texts  = [embedder.build_metadata_text(r) for r in batch]
+    meta_output = embedder.encode_both(meta_texts)
+    meta_embs   = meta_output["dense"]
+    meta_sparse = meta_output["sparse"]
+
+    # Chunk full-text records
+    all_chunks: List[dict] = []
+    for rec in batch:
+        if has_fulltext(rec):
+            all_chunks.extend(chunker.chunk_record(rec))
+
+    # Single forward pass for chunks — dense + sparse together
+    if all_chunks:
+        chunk_texts  = [c["chunk_text"] for c in all_chunks]
+        chunk_output = embedder.encode_both(chunk_texts)
+        chunk_embs   = chunk_output["dense"]
+        chunk_sparse = chunk_output["sparse"]
+    else:
+        chunk_embs   = np.array([])
+        chunk_sparse = []
+
+    with get_conn() as conn:
+        ark_to_doc_id = upsert_documents(batch, meta_embs, meta_sparse, conn)
+        if all_chunks:
+            insert_chunks(all_chunks, chunk_embs, chunk_sparse, ark_to_doc_id, conn)
+
+    return len(batch), len(all_chunks)
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def run_ingestion(
+    fulltext_dir:  str  = DEFAULT_FULLTEXT_DIR,
+    metadata_file: str  = DEFAULT_METADATA_FILE,
+    batch_size:    int  = BGE_BATCH_SIZE,
+    skip_fulltext: bool = False,
+    skip_metadata: bool = False,
+):
+    print(f"\n{'='*60}")
+    print("BPL RAG Ingestion Pipeline")
+    print(f"  Full-text dir  : {fulltext_dir}")
+    print(f"  Metadata file  : {metadata_file}")
+    print(f"  Batch size     : {batch_size}")
+    print(f"{'='*60}\n")
+
+    total_docs = total_chunks = skipped = 0
+    batch: List[dict] = []
+    seen_record_ids: set[str] = set()
+
+    def flush(b):
+        nonlocal total_docs, total_chunks
+        n_docs, n_chunks = flush_batch(b)
+        total_docs   += n_docs
+        total_chunks += n_chunks
+        print(
+            f"  Flushed {n_docs} docs | {n_chunks} chunks | "
+            f"totals → {total_docs} docs / {total_chunks} chunks"
+        )
+
+    if not skip_fulltext:
+        print("── Pass 1: Full-text records ──────────────────────────────")
+        for record in iter_fulltext_records(fulltext_dir):
+            if not is_valid(record):
+                skipped += 1
+                continue
+            seen_record_ids.add(record.get("record_id", ""))
+            batch.append(record)
+            if len(batch) >= batch_size:
+                flush(batch)
+                batch = []
+        if batch:
+            flush(batch)
+            batch = []
+
+    if not skip_metadata:
+        print("\n── Pass 2: Metadata-only records ──────────────────────────")
+        for record in iter_metadata_records(metadata_file, seen_record_ids):
+            if not is_valid(record):
+                skipped += 1
+                continue
+            batch.append(record)
+            if len(batch) >= batch_size:
+                flush(batch)
+                batch = []
+        if batch:
+            flush(batch)
+
+    print(f"\n✓ Ingestion complete.")
+    print(f"  Documents ingested : {total_docs}")
+    print(f"  Chunks created     : {total_chunks}")
+    print(f"  Records skipped    : {skipped}")
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="BPL RAG ingestion pipeline")
+    parser.add_argument("--fulltext-dir",   default=DEFAULT_FULLTEXT_DIR)
+    parser.add_argument("--metadata-file",  default=DEFAULT_METADATA_FILE)
+    parser.add_argument("--batch-size",     type=int, default=BGE_BATCH_SIZE)
+    parser.add_argument("--skip-fulltext",  action="store_true")
+    parser.add_argument("--skip-metadata",  action="store_true")
+    args = parser.parse_args()
+
+    run_ingestion(
+        fulltext_dir  = args.fulltext_dir,
+        metadata_file = args.metadata_file,
+        batch_size    = args.batch_size,
+        skip_fulltext = args.skip_fulltext,
+        skip_metadata = args.skip_metadata,
+    )

pipeline.py

@@ -0,0 +1,110 @@
+"""
+pipeline.py
+
+Top-level entry point wiring all components together.
+Graph retrieval is now fully integrated into retriever.py as a third RRF path.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from dataclasses import dataclass, field
+from typing import List
+
+from retrieval.query_understanding import classify_query, QueryIntent
+from retrieval.retriever import retrieve, RetrievedDocument
+from generation.generator import generate, GenerationResult
+from evaluation.logger import log_query
+from config import TOP_K_FINAL
+
+
+# ── Output dataclass ──────────────────────────────────────────────────────────
+
+@dataclass
+class PipelineResult:
+    intent:     QueryIntent
+    documents:  List[RetrievedDocument]
+    generation: GenerationResult
+    latency_ms: int
+
+
+# ── Main pipeline ─────────────────────────────────────────────────────────────
+
+def run_query(raw_query: str, top_k: int = TOP_K_FINAL) -> PipelineResult:
+    """
+    End-to-end query pipeline:
+      1. Classify and rewrite query (GPT-4o)
+      2. Retrieve via dense + sparse + graph + metadata (all paths, RRF fused)
+      3. Generate cited response (GPT-4o)
+      4. Log event
+    """
+    start = time.monotonic()
+
+    # ── Step 1: Classify ───────────────────────────────────────────────────
+    intent = classify_query(raw_query)
+    print(f"[pipeline] Query type   : {intent.query_type}")
+    print(f"[pipeline] Rewritten    : {intent.rewritten_query}")
+    print(f"[pipeline] Date filter  : {intent.date_filter}")
+
+    # ── Step 2: Retrieve (all paths fused via RRF) ─────────────────────────
+    documents, _ = retrieve(intent, top_k=top_k)
+    print(f"[pipeline] Retrieved    : {len(documents)} documents")
+
+    # ── Step 3: Generate ───────────────────────────────────────────────────
+    if not documents:
+        generation = GenerationResult(
+            response      = "No relevant materials were found for your query in the Digital Commonwealth collection. Try rephrasing or using a more specific historical topic.",
+            source_titles = [],
+            source_urls   = [],
+        )
+    else:
+        generation = generate(raw_query, documents)
+
+    latency_ms = int((time.monotonic() - start) * 1000)
+    print(f"[pipeline] Latency      : {latency_ms}ms")
+
+    # ── Step 4: Log ────────────────────────────────────────────────────────
+    log_query(
+        intent            = intent,
+        retrieved_docs    = documents,
+        generation_result = generation,
+        latency_ms        = latency_ms,
+    )
+
+    return PipelineResult(
+        intent     = intent,
+        documents  = documents,
+        generation = generation,
+        latency_ms = latency_ms,
+    )
+
+
+def print_result(result: PipelineResult):
+    print("\n" + "=" * 60)
+    print("RESPONSE")
+    print("=" * 60)
+    print(result.generation.response)
+
+    print("\n" + "-" * 60)
+    print(f"RESULTS  ({len(result.documents)} docs)")
+    print("-" * 60)
+    for i, doc in enumerate(result.documents, 1):
+        date_str = doc.issue_date or (str(doc.year[0]) if doc.year else "unknown")
+        print(f"  {i}. {doc.title} ({date_str})")
+        print(f"     {doc.source_url}")
+        print(f"     score={doc.final_score:.4f}")
+
+    print(f"\n[latency: {result.latency_ms}ms]")
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python pipeline.py \"<query>\"")
+        sys.exit(1)
+
+    query  = " ".join(sys.argv[1:])
+    result = run_query(query)
+    print_result(result)

requirements.txt

@@ -1,3 +1,84 @@
-altair
-pandas
-streamlit
+# Core dependencies
+streamlit==1.56.0
+python-dotenv==1.2.2
+
+# Database
+psycopg2-binary==2.9.11
+neo4j==6.1.0
+
+# ML/AI - OpenAI & Embeddings
+openai==2.30.0
+sentence-transformers==5.3.0
+transformers==5.5.0
+tokenizers==0.22.2
+tiktoken==0.12.0
+
+# PyTorch (CPU version for HuggingFace)
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.5.1+cpu
+
+# Hugging Face
+huggingface-hub==1.9.0
+accelerate==1.13.0
+safetensors==0.7.0
+datasets==4.8.4
+peft==0.18.1
+
+# NLP - spaCy
+spacy==3.8.14
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl
+
+# Evaluation
+deepeval==3.9.5
+
+# Data processing
+pandas==3.0.2
+numpy==2.4.4
+pyarrow==23.0.1
+scikit-learn==1.8.0
+scipy==1.17.1
+
+# HTTP & Async
+aiohttp==3.13.5
+httpx==0.28.1
+requests==2.33.1
+beautifulsoup4==4.14.3
+
+# Utilities
+pydantic==2.12.5
+pydantic-core==2.41.5
+pydantic-settings==2.13.1
+python-dateutil==2.9.0.post0
+pytz==2026.1.post1
+pyyaml==6.0.3
+click==8.3.2
+tqdm==4.67.3
+filelock==3.25.2
+joblib==1.5.3
+tenacity==9.1.4
+
+# NLP utilities
+blis==1.3.3
+catalogue==2.0.10
+confection==1.3.3
+cymem==2.0.13
+murmurhash==1.0.15
+preshed==3.0.13
+srsly==2.5.3
+thinc==8.3.13
+wasabi==1.1.3
+weasel==1.0.0
+
+# Visualization
+altair==6.0.0
+pillow==12.2.0
+
+# Other
+flagembedding==1.2.5
+inscriptis==2.7.1
+lxml==6.0.2
+regex==2026.4.4
+sentencepiece==0.2.1
+typer==0.24.1

retrieval/query_understanding.py

@@ -0,0 +1,128 @@
+"""
+retrieval/query_understanding.py
+
+Uses GPT-4o to:
+  1. Rewrite the query for better embedding
+  2. Extract hard filters (date range)
+
+GraphRAG is always enabled. No classification into content_driven/metadata_driven.
+Retrieval always runs all paths: chunk search, metadata search, and graph.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Optional
+from openai import OpenAI
+
+from config import OPENAI_API_KEY, OPENAI_CHAT_MODEL
+
+client = OpenAI(api_key=OPENAI_API_KEY)
+
+
+# ── Data structures ───────────────────────────────────────────────────────────
+
+@dataclass
+class DateFilter:
+    year_min: Optional[int] = None
+    year_max: Optional[int] = None
+
+
+@dataclass
+class QueryIntent:
+    raw_query:       str        = ""
+    rewritten_query: str        = ""
+    date_filter:     DateFilter = field(default_factory=DateFilter)
+
+
+# ── System prompt ─────────────────────────────────────────────────────────────
+
+SYSTEM_PROMPT = """
+You are a search assistant for the Boston Public Library's Digital Commonwealth archive.
+The archive contains historical newspapers, photographs, maps, manuscripts, and other
+digitised materials from Massachusetts institutions (1900-1946).
+
+Given a user query, return a JSON object with exactly these fields:
+
+{
+  "rewritten_query": "<clean semantic version of the query for embedding>",
+  "year_min": <integer or null>,
+  "year_max": <integer or null>
+}
+
+Rules for rewritten_query:
+- Expand abbreviations and archaic terms to modern equivalents
+- Remove filler words ("can you find me", "I want to see")
+- Preserve proper nouns, dates, and geographic names exactly
+- Output a concise noun-phrase suitable for semantic embedding
+
+Rules for year_min / year_max:
+- Only set if the user explicitly mentions a time period
+- "1900s" = year_min 1900, year_max 1909
+- "early 20th century" = year_min 1900, year_max 1930
+- Otherwise null
+
+Return ONLY valid JSON. No markdown, no explanation.
+""".strip()
+
+
+# ── Classifier ────────────────────────────────────────────────────────────────
+
+def classify_query(raw_query: str) -> QueryIntent:
+    """
+    Rewrite query and extract date filters.
+    Name kept as classify_query for backward compatibility.
+    """
+    response = client.chat.completions.create(
+        model       = OPENAI_CHAT_MODEL,
+        temperature = 0,
+        max_tokens  = 300,
+        messages    = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user",   "content": raw_query},
+        ],
+    )
+
+    if not response.choices:
+        raise ValueError(f"OpenAI returned empty choices (finish_reason may indicate content filter)")
+    
+    raw_json = response.choices[0].message.content
+    if raw_json is None:
+        raise ValueError("OpenAI returned null content")
+    
+    raw_json = raw_json.strip()
+    
+    if raw_json.startswith("```"):
+        raw_json = raw_json.split("```")[1]
+        if raw_json.startswith("json"):
+            raw_json = raw_json[4:]
+        raw_json = raw_json.strip()
+
+    try:
+        parsed = json.loads(raw_json)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"GPT-4o returned invalid JSON: {e}\nRaw: {raw_json}")
+
+    return QueryIntent(
+        raw_query       = raw_query,
+        rewritten_query = parsed.get("rewritten_query", raw_query),
+        date_filter     = DateFilter(
+            year_min = parsed.get("year_min"),
+            year_max = parsed.get("year_max"),
+        ),
+    )
+
+
+if __name__ == "__main__":
+    test_queries = [
+        "What were some important historical events that happened in Boston in 1919?",
+        "Find pictures of JFK's house on Cape Cod",
+        "Are there any maps of Worcester, MA from the 18th century?",
+        "Who was Mayor Fitzgerald?",
+    ]
+    for q in test_queries:
+        intent = classify_query(q)
+        print(f"\nQuery     : {q}")
+        print(f"Rewritten : {intent.rewritten_query}")
+        print(f"Date      : {intent.date_filter}")

retrieval/retriever.py

@@ -0,0 +1,439 @@
+"""
+retrieval/retriever.py
+
+Hybrid retrieval — three parallel paths fused via RRF:
+  A) Dense search on chunks (full-text)
+  B) Sparse re-rank on dense candidates
+  C) Graph retrieval via Neo4j (entity + co-occurrence)
+  D) Metadata embedding search on documents
+  E) RRF fusion of all result lists
+  F) Final rerank by combined score
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import List, Tuple
+
+import numpy as np
+import time
+
+from config import (
+    TOP_K_DENSE,
+    TOP_K_BM25,
+    TOP_K_FINAL,
+    RRF_K,
+    CONTENT_WEIGHT,
+    METADATA_WEIGHT,
+    MIN_RELEVANCE_SCORE,
+    GRAPH_RAG_ENABLED,
+    GRAPH_TOP_K,
+)
+from database.schema import get_conn, get_cursor
+from embedding.embedder import embedder
+from retrieval.query_understanding import QueryIntent
+
+
+# ── Result dataclass ──────────────────────────────────────────────────────────
+
+@dataclass
+class RetrievedDocument:
+    ark_id:             str
+    document_id:        int
+    title:              str
+    source_url:         str
+    institution:        str
+    issue_date:         str
+    year:               List[int]
+    topics:             List[str]
+    geography:          List[str]
+    best_chunk_text:    str
+    best_chunk_index:   int
+    rrf_score:          float
+    metadata_sim:       float
+    final_score:        float
+    exemplary_image_id: str = ""
+
+
+# ── Filter clause ─────────────────────────────────────────────────────────────
+
+def _build_filter_clause(intent: QueryIntent) -> tuple[str, list]:
+    """Build SQL WHERE clause from date filters only."""
+    conditions = []
+    params     = []
+
+    if intent.date_filter.year_min is not None:
+        conditions.append(
+            "(EXTRACT(YEAR FROM d.date_start) >= %s OR d.date_start IS NULL)"
+        )
+        params.append(float(intent.date_filter.year_min))
+
+    if intent.date_filter.year_max is not None:
+        conditions.append(
+            "(EXTRACT(YEAR FROM d.date_start) <= %s OR d.date_start IS NULL)"
+        )
+        params.append(float(intent.date_filter.year_max))
+
+    where = "WHERE " + " AND ".join(conditions) if conditions else ""
+    return where, params
+
+
+# ── Dense chunk search ────────────────────────────────────────────────────────
+
+def _dense_search(
+    query_embedding: np.ndarray,
+    where_clause: str,
+    where_params: list,
+    conn,
+    top_k: int = TOP_K_DENSE,
+) -> List[dict]:
+    """HNSW vector search on chunk embeddings with post-filter date constraints."""
+    emb_list = query_embedding.tolist()
+
+    # Build date filter for application AFTER HNSW search
+    date_filter = ""
+    if where_clause:
+        # Strip "WHERE " and keep the rest
+        date_filter = "WHERE " + where_clause.replace("WHERE ", "")
+    
+    sql = f"""
+        WITH top_chunks AS (
+            -- Step 1: HNSW search unrestricted - fast
+            SELECT
+                c.document_id,
+                c.id           AS chunk_id,
+                c.chunk_index,
+                c.chunk_text,
+                c.sparse_token_ids,
+                c.sparse_weights,
+                (c.text_embedding <=> %s::vector) AS distance
+            FROM chunks c
+            ORDER BY c.text_embedding <=> %s::vector
+            LIMIT %s
+        ),
+        nearest_chunks AS (
+            -- Step 2: Deduplicate to best chunk per document
+            SELECT DISTINCT ON (document_id)
+                *,
+                1 - distance AS dense_score
+            FROM top_chunks
+            ORDER BY document_id, distance
+        )
+        SELECT
+            nc.*,
+            d.ark_id, d.title, d.source_url, d.institution,
+            d.issue_date, d.year, d.topics, d.geography,
+            d.metadata_embedding,
+            d.exemplary_image_id,
+            d.sparse_token_ids AS doc_sparse_token_ids,
+            d.sparse_weights   AS doc_sparse_weights
+        FROM nearest_chunks nc
+        JOIN documents d ON d.id = nc.document_id
+        {date_filter}
+        ORDER BY nc.dense_score DESC
+    """
+
+    # HNSW fetches top_k * 3 chunks, then date filter is applied after join
+    params = [emb_list, emb_list, top_k * 3] + where_params
+
+    with get_cursor(conn) as cur:
+        cur.execute(sql, params)
+        return cur.fetchall()
+
+# ── Sparse re-ranking on dense candidates ────────────────────────────────────
+
+def _sparse_search(
+    query_sparse: dict,
+    dense_results: List[dict],
+    top_k: int = TOP_K_BM25,
+) -> List[dict]:
+    """Score sparse on dense candidates only — no extra DB call."""
+    if not query_sparse or not dense_results:
+        return []
+
+    scored = []
+    for row in dense_results:
+        token_ids    = row.get("sparse_token_ids") or []
+        weights      = row.get("sparse_weights")   or []
+        chunk_sparse = dict(zip([str(t) for t in token_ids], weights))
+        score = sum(
+            float(query_sparse.get(tok, 0.0)) * float(weight)
+            for tok, weight in chunk_sparse.items()
+        )
+        row_dict = dict(row)
+        row_dict["bm25_score"] = score
+        scored.append(row_dict)
+
+    scored.sort(key=lambda x: x["bm25_score"], reverse=True)
+    return scored[:top_k]
+
+
+# ── Metadata document search ──────────────────────────────────────────────────
+
+def _metadata_search(
+    query_embedding: np.ndarray,
+    where_clause: str,
+    where_params: list,
+    conn,
+    top_k: int = TOP_K_DENSE,
+) -> List[dict]:
+    """Search all documents by metadata embedding similarity."""
+    emb_list = query_embedding.tolist()
+
+    sql = f"""
+        SELECT
+            d.id AS document_id,
+            d.ark_id,
+            d.title,
+            d.source_url,
+            d.institution,
+            d.issue_date,
+            d.year,
+            d.topics,
+            d.geography,
+            d.metadata_embedding,
+            d.exemplary_image_id,
+            d.sparse_token_ids AS doc_sparse_token_ids,
+            d.sparse_weights   AS doc_sparse_weights,
+            '' AS chunk_text,
+            0  AS chunk_index,
+            1 - (d.metadata_embedding <=> %s::vector) AS dense_score
+        FROM documents d
+        {where_clause}
+        ORDER BY d.metadata_embedding <=> %s::vector
+        LIMIT %s
+    """
+
+    params = [emb_list] + where_params + [emb_list, top_k]
+
+    with get_cursor(conn) as cur:
+        cur.execute(sql, params)
+        return cur.fetchall()
+
+
+# ── Graph retrieval ───────────────────────────────────────────────────────────
+
+def _graph_search(
+    query_embedding: np.ndarray,
+    exclude_ark_ids: set,
+    conn,
+    top_k: int = GRAPH_TOP_K,
+) -> List[dict]:
+    """
+    Graph retrieval via Neo4j entity matching + two-hop traversal.
+    Fetches best chunk per document by embedding similarity from PostgreSQL.
+    Returns results in the same dict format as dense/metadata search for RRF.
+    """
+    from graph.graph_retriever import retrieve_by_query
+
+    graph_results = retrieve_by_query(
+        query_embedding = query_embedding,
+        exclude_ark_ids = exclude_ark_ids,
+        top_k           = top_k,
+    )
+
+    if not graph_results:
+        print("[retriever] Graph returned no results")
+        return []
+
+    # Fetch full document details + best chunk by embedding similarity
+    graph_ark_ids = [r.ark_id for r in graph_results]
+    emb_list      = query_embedding.tolist()
+
+    sql = """
+        SELECT
+            d.id AS document_id,
+            d.ark_id,
+            d.title,
+            d.source_url,
+            d.institution,
+            d.issue_date,
+            d.year,
+            d.topics,
+            d.geography,
+            d.metadata_embedding,
+            d.exemplary_image_id,
+            d.sparse_token_ids AS doc_sparse_token_ids,
+            d.sparse_weights   AS doc_sparse_weights,
+            c.chunk_text,
+            c.chunk_index
+        FROM documents d
+        LEFT JOIN LATERAL (
+            SELECT chunk_text, chunk_index
+            FROM chunks
+            WHERE document_id = d.id
+            ORDER BY text_embedding <=> %s::vector
+            LIMIT 1
+        ) c ON true
+        WHERE d.ark_id = ANY(%s)
+    """
+
+    with get_cursor(conn) as cur:
+        cur.execute(sql, (emb_list, graph_ark_ids))
+        rows = {row["ark_id"]: row for row in cur.fetchall()}
+
+    # Build result dicts in same format as dense/metadata results
+    results = []
+    for graph_result in graph_results:
+        row = rows.get(graph_result.ark_id)
+        if not row:
+            continue
+        result = dict(row)
+        result["graph_score"] = graph_result.graph_score
+        results.append(result)
+
+    return results
+
+
+# ── RRF fusion ────────────────────────────────────────────────────────────────
+
+def _reciprocal_rank_fusion(
+    *result_lists: List[dict],
+    k: int = RRF_K,
+) -> List[tuple[str, float, dict]]:
+    """
+    Merge any number of ranked lists using RRF.
+    Each list contributes 1/(k + rank) to the score.
+    Empty lists are skipped silently.
+    """
+    scores: dict[str, float] = {}
+    rows:   dict[str, dict]  = {}
+
+    for result_list in result_lists:
+        if not result_list:
+            continue
+        for rank, row in enumerate(result_list, start=1):
+            aid = row["ark_id"]
+            scores[aid] = scores.get(aid, 0.0) + 1.0 / (k + rank)
+            if aid not in rows:
+                rows[aid] = row
+
+    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+    return [(aid, score, rows[aid]) for aid, score in ranked]
+
+
+# ── Final rerank ──────────────────────────────────────────────────────────────
+
+def _rerank(
+    rrf_results:     List[tuple[str, float, dict]],
+    query_embedding: np.ndarray,
+    top_k:           int = TOP_K_FINAL,
+) -> List[RetrievedDocument]:
+    """
+    Final rerank blending RRF score with metadata embedding similarity.
+    """
+    if not rrf_results:
+        return []
+
+    final: List[RetrievedDocument] = []
+
+    for ark_id, rrf_score, row in rrf_results:
+        meta_emb = row.get("metadata_embedding")
+        if meta_emb is not None:
+            if isinstance(meta_emb, str):
+                meta_emb = json.loads(meta_emb)
+            meta_vec = np.array(meta_emb, dtype=np.float32)
+            meta_sim = float(np.dot(query_embedding, meta_vec))
+            meta_sim = max(0.0, min(1.0, meta_sim))
+        else:
+            meta_sim = 0.0
+
+        final_score = CONTENT_WEIGHT * rrf_score + METADATA_WEIGHT * meta_sim
+
+        final.append(RetrievedDocument(
+            ark_id             = ark_id,
+            document_id        = row["document_id"],
+            title              = row.get("title", ""),
+            source_url         = row.get("source_url", ""),
+            institution        = row.get("institution", ""),
+            issue_date         = row.get("issue_date", ""),
+            year               = row.get("year") or [],
+            topics             = row.get("topics") or [],
+            geography          = row.get("geography") or [],
+            best_chunk_text    = row.get("chunk_text", ""),
+            best_chunk_index   = row.get("chunk_index", 0),
+            exemplary_image_id = row.get("exemplary_image_id") or "",
+            rrf_score          = rrf_score,
+            metadata_sim       = meta_sim,
+            final_score        = final_score,
+        ))
+
+    final.sort(key=lambda x: x.final_score, reverse=True)
+    final = [doc for doc in final if doc.final_score >= MIN_RELEVANCE_SCORE]
+    return final[:top_k]
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+def retrieve(intent: QueryIntent, top_k: int = TOP_K_FINAL) -> Tuple[List[RetrievedDocument], np.ndarray]:
+    """
+    Full hybrid retrieval:
+      1. Build date filter from intent
+      2. Embed query (dense + sparse in one pass)
+      3. Dense chunk search (HNSW)
+      4. Sparse re-rank on dense candidates
+      5. Metadata document search
+      6. Graph retrieval via Neo4j (if enabled)
+      7. RRF fusion of all result lists
+      8. Final rerank and threshold filter
+
+    Returns (documents, query_embedding) — embedding returned to avoid
+    re-computation in downstream callers.
+    """
+    where_clause, where_params = _build_filter_clause(intent)
+
+    query_output = embedder.encode_one_both(intent.rewritten_query, is_query=True)
+    query_emb    = query_output["dense"]
+    query_sparse = query_output["sparse"]
+
+    with get_conn() as conn:
+        # Path A: chunk-level dense search
+        t = time.monotonic()
+        print("[retrieve] starting dense", flush=True)
+        print(f"[retrieve] where_clause: '{where_clause}'", flush=True)
+        print(f"[retrieve] where_params: {where_params}", flush=True)
+        dense_results = _dense_search(query_emb, where_clause, where_params, conn)
+        dense_results  = _dense_search(
+            query_emb, where_clause, where_params, conn
+        )
+        print(f"[timing] dense: {time.monotonic()-t:.2f}s", flush=True)
+        # Path B: sparse re-rank on dense candidates
+        t = time.monotonic()
+        print("[retrieve] starting sparse", flush=True)
+        sparse_results = _sparse_search(
+            query_sparse, dense_results
+        )
+       
+        # Path C: document-level metadata search
+        t = time.monotonic()
+        print("[retrieve] starting meta", flush=True)
+        meta_results = _metadata_search(
+            query_emb, where_clause, where_params, conn,
+            top_k=TOP_K_DENSE,
+        )
+        print(f"[timing] meta: {time.monotonic()-t:.2f}s", flush=True)
+        # Path D: graph retrieval
+        graph_results = []
+        if GRAPH_RAG_ENABLED:
+            existing_ark_ids = {r["ark_id"] for r in dense_results + meta_results}
+            t = time.monotonic()
+            print("[retrieve] starting graph", flush=True)
+            graph_results = _graph_search(
+                query_emb,
+                exclude_ark_ids = existing_ark_ids,
+                conn            = conn,
+                top_k           = GRAPH_TOP_K,
+            )
+            print(f"[timing] graph: {time.monotonic()-t:.2f}s", flush=True)
+
+
+    # Fuse all paths via RRF
+    rrf_results = _reciprocal_rank_fusion(
+        dense_results,
+        sparse_results,
+        meta_results,
+        graph_results,
+    )
+
+    return _rerank(rrf_results, query_emb, top_k=top_k), query_emb

scripts/update_abstracts_and_embeddings.py

@@ -0,0 +1,221 @@
+"""
+scripts/update_abstracts_and_embeddings.py
+
+For all metadata-only documents that have a non-empty abstract in the
+JSONL source file:
+  1. Updates the `abstract` column in PostgreSQL with the full text
+  2. Reconstructs the metadata text string (identical to ingestion)
+  3. Re-computes dense + sparse embeddings using BGE-M3
+  4. Updates metadata_embedding, sparse_token_ids, sparse_weights in DB
+
+Run:
+    python scripts/update_abstracts_and_embeddings.py
+    python scripts/update_abstracts_and_embeddings.py --metadata-file data/metadata/metadata.jsonl
+"""
+
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import re
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List
+
+# Add project root to path so imports work when run as script
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from database.schema import get_conn, get_cursor
+from embedding.embedder import embedder
+
+DEFAULT_METADATA_FILE = "data/metadata/metadata.jsonl"
+BATCH_SIZE = 64
+
+
+# ── HTML cleaning (identical to _parse_metadata_record) ───────────────────────
+
+def clean_abstract(raw: str) -> str:
+    """Strip HTML exactly as _parse_metadata_record does — no truncation."""
+    text = html.unescape(raw)
+    text = re.sub(r"<[^>]+>", " ", text).strip()
+    text = re.sub(r"\s+", " ", text)
+    return text
+
+
+# ── Build metadata text (identical to build_metadata_text in embedder.py) ─────
+
+def build_metadata_text(record: dict) -> str:
+    """
+    Reconstruct the metadata text string exactly as embedder.build_metadata_text()
+    does — with abstract truncation removed.
+    """
+    parts = []
+
+    if record.get("title"):
+        parts.append(f"Title: {record['title']}")
+
+    genres = record.get("genre") or []
+    if genres:
+        parts.append(f"Format: {', '.join(genres)}")
+
+    topics = record.get("topics") or []
+    if topics:
+        parts.append(f"Topics: {', '.join(topics)}")
+
+    geography = record.get("geography") or []
+    if geography:
+        parts.append(f"Geography: {', '.join(geography)}")
+
+    place = record.get("place") or []
+    if place:
+        parts.append(f"Place: {', '.join(place)}")
+
+    year = record.get("year") or []
+    if year:
+        parts.append(f"Year: {', '.join(str(y) for y in year)}")
+
+    collection = record.get("collection") or ""
+    if collection and collection != record.get("title"):
+        parts.append(f"Collection: {collection}")
+
+    abstract = record.get("abstract") or ""
+    if abstract:
+        # No truncation — full abstract
+        parts.append(f"Description: {abstract}")
+
+    return " | ".join(parts)
+
+
+# ── Update DB ─────────────────────────────────────────────────────────────────
+
+def update_batch(batch: List[dict]):
+    """
+    For a batch of records with full abstracts:
+    1. Compute new metadata text
+    2. Embed dense + sparse in one forward pass
+    3. Update abstract, metadata_embedding, sparse_token_ids, sparse_weights
+    """
+    meta_texts = [build_metadata_text(r) for r in batch]
+
+    output     = embedder.encode_both(meta_texts)
+    dense_embs = output["dense"]
+    sparse_embs = output["sparse"]
+
+    with get_conn() as conn:
+        for rec, dense_emb, sparse in zip(batch, dense_embs, sparse_embs):
+            token_ids = [int(k) for k in sparse.keys()]
+            weights   = [float(v) for v in sparse.values()]
+
+            with get_cursor(conn) as cur:
+                cur.execute(
+                    """
+                    UPDATE documents
+                    SET
+                        abstract           = %s,
+                        metadata_embedding = %s,
+                        sparse_token_ids   = %s,
+                        sparse_weights     = %s,
+                        ingested_at        = %s
+                    WHERE ark_id = %s
+                    """,
+                    (
+                        rec["abstract"],
+                        dense_emb.tolist(),
+                        token_ids,
+                        weights,
+                        datetime.now(timezone.utc).isoformat(),
+                        rec["ark_id"],
+                    )
+                )
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def run(metadata_file: str = DEFAULT_METADATA_FILE):
+    fpath = Path(metadata_file)
+    if not fpath.exists():
+        print(f"File not found: {metadata_file}")
+        return
+
+    print(f"\n{'='*60}")
+    print("Update Abstracts + Embeddings for Metadata Records")
+    print(f"  Source : {metadata_file}")
+    print(f"  Batch  : {BATCH_SIZE}")
+    print(f"{'='*60}\n")
+
+    print("Reading metadata JSONL...")
+    records_to_update = []
+
+    with open(fpath, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                raw = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            data      = raw.get("data", {})
+            attrs     = data.get("attributes", {})
+            record_id = data.get("id", "")
+            ark_id    = record_id.split(":")[-1] if ":" in record_id else record_id
+
+            abstract_raw = attrs.get("abstract_tsi", "") or ""
+            if not abstract_raw.strip():
+                continue
+
+            abstract = clean_abstract(abstract_raw)
+            if not abstract:
+                continue
+
+            records_to_update.append({
+                "ark_id":      ark_id,
+                "abstract":    abstract,
+                "genre":       attrs.get("genre_basic_ssim", []),
+                "topics":      attrs.get("subject_topic_tsim", []),
+                "geography":   attrs.get("subject_geographic_ssim", []),
+                "place":       attrs.get("publication_place_tsim", []),
+                "year":        attrs.get("date_facet_yearly_itim", []),
+                "title":       attrs.get("title_info_primary_tsi", ""),
+                "collection":  attrs.get("title_info_primary_tsi", ""),
+                "institution": attrs.get("institution_name_ssi", ""),
+            })
+
+    print(f"  Found {len(records_to_update)} records with non-empty abstracts\n")
+
+    if not records_to_update:
+        print("Nothing to update.")
+        return
+
+    total_updated = 0
+    start_time    = time.monotonic()
+
+    for i in range(0, len(records_to_update), BATCH_SIZE):
+        batch = records_to_update[i:i + BATCH_SIZE]
+        update_batch(batch)
+        total_updated += len(batch)
+
+        elapsed   = time.monotonic() - start_time
+        remaining = (elapsed / total_updated) * (len(records_to_update) - total_updated) if total_updated else 0
+        print(
+            f"  [{total_updated}/{len(records_to_update)}] "
+            f"ETA: {remaining/60:.1f}min"
+        )
+
+    print(f"\n✓ Done.")
+    print(f"  Records updated : {total_updated}")
+    print(f"  Total time      : {(time.monotonic()-start_time)/60:.1f} min")
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--metadata-file", default=DEFAULT_METADATA_FILE)
+    args = parser.parse_args()
+    run(metadata_file=args.metadata_file)