feat: add progress tracking with docs/pages/mentions verification stats

- /api/progress endpoint computes per-doc and overall stats
- ProgressBar component: collapsible bar showing verification %
- Expanded view: colored progress bars for docs, pages, mentions
- Shows count of human annotations

app/api/progress/route.js

@@ -0,0 +1,115 @@
+import { HF_DATASET_BASE_URL, MAX_DOCS_TO_SCAN } from '../../../utils/config.js';
+
+/**
+ * GET /api/progress
+ * Returns progress stats: total docs, pages, mentions, and how many are verified.
+ */
+export async function GET() {
+    try {
+        const linksUrl = `${HF_DATASET_BASE_URL}/raw/main/annotation_data/wbg_data/wbg_pdf_links.json`;
+        const linksRes = await fetch(linksUrl, {
+            headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` },
+            next: { revalidate: 300 } // cache 5 min
+        });
+
+        if (!linksRes.ok) {
+            return new Response(JSON.stringify({ error: 'Failed to fetch links' }), { status: 500 });
+        }
+
+        const links = await linksRes.json();
+        const activeLinks = links
+            .filter(l => l.status === 'success' && l.has_revalidation === true)
+            .slice(0, MAX_DOCS_TO_SCAN);
+
+        // Fetch all docs in parallel
+        const results = await Promise.allSettled(
+            activeLinks.map(async (link) => {
+                const docUrl = `${HF_DATASET_BASE_URL}/raw/main/annotation_data/wbg_extractions/doc_${link.index}/raw/doc_${link.index}_direct_judged.jsonl`;
+                const docRes = await fetch(docUrl, {
+                    headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` }
+                });
+                if (!docRes.ok) return null;
+
+                const pagesData = await docRes.json();
+
+                let totalMentions = 0;
+                let verifiedMentions = 0;
+                let totalPages = 0;
+                let completedPages = 0;
+                let humanAnnotations = 0;
+
+                for (const page of pagesData) {
+                    const datasets = (page.datasets || []).filter(ds => {
+                        // Exclude consensus non-datasets
+                        if (ds.dataset_tag === 'non-dataset' && ds.dataset_name?.judge_agrees === true) {
+                            return false;
+                        }
+                        return true;
+                    });
+
+                    if (datasets.length === 0) continue;
+
+                    totalPages++;
+                    totalMentions += datasets.length;
+
+                    let pageVerified = 0;
+                    for (const ds of datasets) {
+                        if (ds.human_validated === true) {
+                            verifiedMentions++;
+                            pageVerified++;
+                        }
+                        if (ds.source === 'human') {
+                            humanAnnotations++;
+                        }
+                    }
+
+                    // A page is "completed" if all its mentions are verified
+                    if (pageVerified === datasets.length) {
+                        completedPages++;
+                    }
+                }
+
+                const docComplete = totalPages > 0 && completedPages === totalPages;
+
+                return {
+                    index: link.index,
+                    totalPages,
+                    completedPages,
+                    totalMentions,
+                    verifiedMentions,
+                    humanAnnotations,
+                    complete: docComplete,
+                };
+            })
+        );
+
+        const docs = results
+            .filter(r => r.status === 'fulfilled' && r.value !== null)
+            .map(r => r.value);
+
+        const summary = {
+            totalDocs: docs.length,
+            completedDocs: docs.filter(d => d.complete).length,
+            totalPages: docs.reduce((s, d) => s + d.totalPages, 0),
+            completedPages: docs.reduce((s, d) => s + d.completedPages, 0),
+            totalMentions: docs.reduce((s, d) => s + d.totalMentions, 0),
+            verifiedMentions: docs.reduce((s, d) => s + d.verifiedMentions, 0),
+            humanAnnotations: docs.reduce((s, d) => s + d.humanAnnotations, 0),
+            docs,
+        };
+
+        return new Response(JSON.stringify(summary), {
+            status: 200,
+            headers: {
+                'Content-Type': 'application/json',
+                'Cache-Control': 'public, s-maxage=300, stale-while-revalidate=59'
+            }
+        });
+    } catch (error) {
+        console.error('Progress API error:', error);
+        return new Response(
+            JSON.stringify({ error: 'Failed to compute progress' }),
+            { status: 500, headers: { 'Content-Type': 'application/json' } }
+        );
+    }
+}

app/components/ProgressBar.js

@@ -0,0 +1,83 @@
+"use client";
+
+import { useState, useEffect } from 'react';
+
+export default function ProgressBar() {
+    const [progress, setProgress] = useState(null);
+    const [expanded, setExpanded] = useState(false);
+    const [loading, setLoading] = useState(true);
+
+    useEffect(() => {
+        fetch('/api/progress')
+            .then(res => res.json())
+            .then(data => {
+                setProgress(data);
+                setLoading(false);
+            })
+            .catch(() => setLoading(false));
+    }, []);
+
+    if (loading || !progress) {
+        return null; // Don't show anything while loading
+    }
+
+    const docPct = progress.totalDocs > 0
+        ? Math.round((progress.completedDocs / progress.totalDocs) * 100) : 0;
+    const pagePct = progress.totalPages > 0
+        ? Math.round((progress.completedPages / progress.totalPages) * 100) : 0;
+    const mentionPct = progress.totalMentions > 0
+        ? Math.round((progress.verifiedMentions / progress.totalMentions) * 100) : 0;
+
+    return (
+        <div className="progress-container">
+            <button
+                className="progress-toggle"
+                onClick={() => setExpanded(!expanded)}
+            >
+                <span className="progress-summary">
+                    📊 Progress: {progress.verifiedMentions}/{progress.totalMentions} mentions verified ({mentionPct}%)
+                </span>
+                <span className="progress-chevron">{expanded ? '▲' : '▼'}</span>
+            </button>
+
+            {expanded && (
+                <div className="progress-details">
+                    <div className="progress-row">
+                        <span className="progress-label">📄 Documents</span>
+                        <div className="progress-bar-track">
+                            <div
+                                className="progress-bar-fill docs-fill"
+                                style={{ width: `${docPct}%` }}
+                            />
+                        </div>
+                        <span className="progress-stat">{progress.completedDocs}/{progress.totalDocs} ({docPct}%)</span>
+                    </div>
+                    <div className="progress-row">
+                        <span className="progress-label">📑 Pages</span>
+                        <div className="progress-bar-track">
+                            <div
+                                className="progress-bar-fill pages-fill"
+                                style={{ width: `${pagePct}%` }}
+                            />
+                        </div>
+                        <span className="progress-stat">{progress.completedPages}/{progress.totalPages} ({pagePct}%)</span>
+                    </div>
+                    <div className="progress-row">
+                        <span className="progress-label">🏷️ Mentions</span>
+                        <div className="progress-bar-track">
+                            <div
+                                className="progress-bar-fill mentions-fill"
+                                style={{ width: `${mentionPct}%` }}
+                            />
+                        </div>
+                        <span className="progress-stat">{progress.verifiedMentions}/{progress.totalMentions} ({mentionPct}%)</span>
+                    </div>
+                    <div className="progress-row">
+                        <span className="progress-label">✍️ Human annotations</span>
+                        <span className="progress-stat highlight">{progress.humanAnnotations}</span>
+                    </div>
+                </div>
+            )}
+        </div>
+    );
+}

app/globals.css

@@ -158,6 +158,100 @@ h4 {
   overflow: hidden;
 }
 
+/* ── Progress Bar ────────────────────────────── */
+
+.progress-container {
+  border-bottom: 1px solid var(--border-color);
+  background: var(--pane-bg);
+  flex-shrink: 0;
+}
+
+.progress-toggle {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  width: 100%;
+  padding: 5px 20px;
+  background: none;
+  border: none;
+  color: var(--text-color);
+  cursor: pointer;
+  font-size: 0.78rem;
+}
+
+.progress-toggle:hover {
+  background: rgba(255, 255, 255, 0.03);
+}
+
+.progress-summary {
+  font-weight: 600;
+}
+
+.progress-chevron {
+  font-size: 0.6rem;
+  color: #64748b;
+}
+
+.progress-details {
+  padding: 8px 20px 10px;
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+  border-top: 1px solid var(--border-color);
+}
+
+.progress-row {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  font-size: 0.75rem;
+}
+
+.progress-label {
+  width: 130px;
+  flex-shrink: 0;
+  color: #94a3b8;
+}
+
+.progress-bar-track {
+  flex: 1;
+  height: 6px;
+  background: var(--surface);
+  border-radius: 3px;
+  overflow: hidden;
+}
+
+.progress-bar-fill {
+  height: 100%;
+  border-radius: 3px;
+  transition: width 0.5s ease;
+}
+
+.docs-fill {
+  background: var(--success);
+}
+
+.pages-fill {
+  background: var(--accent);
+}
+
+.mentions-fill {
+  background: #f59e0b;
+}
+
+.progress-stat {
+  font-size: 0.72rem;
+  color: #94a3b8;
+  white-space: nowrap;
+  min-width: 80px;
+  text-align: right;
+}
+
+.progress-stat.highlight {
+  color: var(--success);
+  font-weight: 600;
+}
+
 .pane {
   flex: 1;
   padding: 20px 24px;

app/page.js

@@ -7,6 +7,7 @@ import MarkdownAnnotator from './components/MarkdownAnnotator';
 import AnnotationPanel from './components/AnnotationPanel';
 import AnnotationModal from './components/AnnotationModal';
 import PageNavigator from './components/PageNavigator';
+import ProgressBar from './components/ProgressBar';
 
 export default function Home() {
     const [documents, setDocuments] = useState([]);
@@ -430,6 +431,7 @@ export default function Home() {
                     )}
                 </div>
             </div>
+            <ProgressBar />
             <div className="container">
                 <div className="pane left-pane">
                     <div className="pane-header">