feat: on-demand document loading, no limit, PDF loading indicator

- /api/documents now returns just the link list (no pre-fetching)
- /api/document returns annotatable_pages on-demand (no page param)
- page.js fetches + caches annotatable_pages when doc is selected
- PdfViewer shows loading spinner overlay while iframe loads
- Removed MAX_DOCS_TO_SCAN concept entirely

app/api/document/route.js

@@ -6,67 +6,80 @@ const isHFSpace = () => {
     return process.env.HF_TOKEN && process.env.NODE_ENV !== 'development';
 };
 
+/**
+ * Loads ALL pages data for a given document index.
+ */
+async function loadPagesData(indexNum) {
+    if (isHFSpace()) {
+        const docUrl = `${HF_DATASET_BASE_URL}/raw/main/annotation_data/wbg_extractions/doc_${indexNum}/raw/doc_${indexNum}_direct_judged.jsonl`;
+        const res = await fetch(docUrl, {
+            headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` }
+        });
+        if (!res.ok) return null;
+        return await res.json();
+    } else {
+        const filePath = path.join(
+            process.cwd(),
+            'annotation_data', 'wbg_extractions',
+            `doc_${indexNum}`, 'raw', `doc_${indexNum}_direct_judged.jsonl`
+        );
+        if (!fs.existsSync(filePath)) return null;
+        return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+    }
+}
+
 export async function GET(request) {
     const { searchParams } = new URL(request.url);
     const index = searchParams.get('index');
     const page = searchParams.get('page');
 
-    // Validate required params
-    if (index === null || page === null) {
+    if (index === null) {
         return new Response(
-            JSON.stringify({ error: "Missing index or page parameter" }),
+            JSON.stringify({ error: "Missing index parameter" }),
             { status: 400, headers: { 'Content-Type': 'application/json' } }
         );
     }
 
-    // Validate numeric values
     const indexNum = parseInt(index, 10);
-    const pageNum = parseInt(page, 10);
-
-    if (isNaN(indexNum) || isNaN(pageNum) || indexNum < 0 || pageNum < 0) {
+    if (isNaN(indexNum) || indexNum < 0) {
         return new Response(
-            JSON.stringify({ error: "index and page must be non-negative integers" }),
+            JSON.stringify({ error: "index must be a non-negative integer" }),
             { status: 400, headers: { 'Content-Type': 'application/json' } }
         );
     }
 
     try {
-        let pagesData;
+        const pagesData = await loadPagesData(indexNum);
+
+        if (!pagesData) {
+            return new Response(
+                JSON.stringify({ error: `doc_${indexNum}_direct_judged.jsonl not found` }),
+                { status: 404, headers: { 'Content-Type': 'application/json' } }
+            );
+        }
+
+        // If no page specified, return just the annotatable pages list
+        if (page === null || page === undefined) {
+            const annotatablePages = pagesData
+                .filter(p => p.datasets && p.datasets.length > 0)
+                .map(p => p.document?.pages?.[0]);
 
-        if (isHFSpace()) {
-            // Production: fetch from HuggingFace
-            const docUrl = `${HF_DATASET_BASE_URL}/raw/main/annotation_data/wbg_extractions/doc_${indexNum}/raw/doc_${indexNum}_direct_judged.jsonl`;
-            const res = await fetch(docUrl, {
-                headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` }
+            return new Response(JSON.stringify({ annotatable_pages: annotatablePages }), {
+                status: 200,
+                headers: { 'Content-Type': 'application/json' }
             });
+        }
 
-            if (!res.ok) {
-                return new Response(
-                    JSON.stringify({ error: `doc_${indexNum}_direct_judged.jsonl not found on HF Datasets` }),
-                    { status: res.status, headers: { 'Content-Type': 'application/json' } }
-                );
-            }
-            pagesData = await res.json();
-        } else {
-            // Local dev: read from local file (reflects saved annotations immediately)
-            const filePath = path.join(
-                process.cwd(),
-                'annotation_data', 'wbg_extractions',
-                `doc_${indexNum}`, 'raw', `doc_${indexNum}_direct_judged.jsonl`
+        // Specific page requested
+        const pageNum = parseInt(page, 10);
+        if (isNaN(pageNum) || pageNum < 0) {
+            return new Response(
+                JSON.stringify({ error: "page must be a non-negative integer" }),
+                { status: 400, headers: { 'Content-Type': 'application/json' } }
             );
-
-            if (!fs.existsSync(filePath)) {
-                return new Response(
-                    JSON.stringify({ error: `doc_${indexNum}_direct_judged.jsonl not found locally` }),
-                    { status: 404, headers: { 'Content-Type': 'application/json' } }
-                );
-            }
-            const raw = fs.readFileSync(filePath, 'utf-8');
-            pagesData = JSON.parse(raw);
         }
 
         const pageData = pagesData.find(p => p.document?.pages?.[0] === pageNum);
-
         if (!pageData) {
             return new Response(
                 JSON.stringify({ error: `Page ${pageNum} not found in doc ${indexNum}` }),

app/api/documents/route.js

@@ -1,4 +1,4 @@
-import { HF_DATASET_BASE_URL, MAX_DOCS_TO_SCAN } from '../../../utils/config.js';
+import { HF_DATASET_BASE_URL } from '../../../utils/config.js';
 
 export async function GET() {
     try {
@@ -21,38 +21,17 @@ export async function GET() {
 
         const links = await linksRes.json();
 
-        // Filter to successful links and take the first N
-        const successLinks = links.filter(l => l.status === 'success').slice(0, MAX_DOCS_TO_SCAN);
-
-        // Parallel fetch — much faster than sequential scanning
-        const results = await Promise.allSettled(
-            successLinks.map(async (link) => {
-                const docUrl = `${HF_DATASET_BASE_URL}/raw/main/annotation_data/wbg_extractions/doc_${link.index}/raw/doc_${link.index}_direct_judged.jsonl`;
-                const docRes = await fetch(docUrl, {
-                    headers: { 'Authorization': `Bearer ${process.env.HF_TOKEN}` }
-                });
-
-                if (!docRes.ok) return null;
-
-                const pagesData = await docRes.json();
-                const annotatablePages = pagesData
-                    .filter(page => page.datasets && page.datasets.length > 0)
-                    .map(page => page.document.pages[0]);
-
-                if (annotatablePages.length === 0) return null;
-
-                return {
-                    index: link.index,
-                    pdf_url: link.direct_pdf_url,
-                    landing_page: link.landing_page_url,
-                    annotatable_pages: annotatablePages
-                };
-            })
-        );
-
-        const documents = results
-            .filter(r => r.status === 'fulfilled' && r.value !== null)
-            .map(r => r.value);
+        // Return ALL successful links — no limit, no data pre-fetching.
+        // Page data is loaded on-demand when the user selects a document.
+        const documents = links
+            .filter(l => l.status === 'success')
+            .map(link => ({
+                index: link.index,
+                pdf_url: link.direct_pdf_url,
+                landing_page: link.landing_page_url,
+                // annotatable_pages will be fetched on-demand via /api/document
+                annotatable_pages: null,
+            }));
 
         return new Response(JSON.stringify(documents), {
             status: 200,

app/components/PdfViewer.js

@@ -1,6 +1,10 @@
 "use client";
 
+import { useState } from 'react';
+
 export default function PdfViewer({ pdfUrl, pageNumber }) {
+    const [pdfLoading, setPdfLoading] = useState(true);
+
     if (!pdfUrl) {
         return (
             <div className="pdf-placeholder">
@@ -13,16 +17,24 @@ export default function PdfViewer({ pdfUrl, pageNumber }) {
     const viewerPage = (pageNumber ?? 0) + 1;
 
     // Use Mozilla's hosted PDF.js viewer — supports #page=N for direct page navigation.
-    // This avoids X-Frame-Options restrictions from the source server.
     const pdfJsViewerUrl = `https://mozilla.github.io/pdf.js/web/viewer.html?file=${encodeURIComponent(pdfUrl)}#page=${viewerPage}`;
 
     return (
-        <iframe
-            key={`pdf-${pdfUrl}-page-${viewerPage}`}
-            src={pdfJsViewerUrl}
-            className="pdf-frame"
-            title={`PDF Page ${viewerPage}`}
-            allow="fullscreen"
-        />
+        <div className="pdf-container">
+            {pdfLoading && (
+                <div className="pdf-loading-overlay">
+                    <div className="loading-spinner" />
+                    <p>Loading PDF...</p>
+                </div>
+            )}
+            <iframe
+                key={`pdf-${pdfUrl}-page-${viewerPage}`}
+                src={pdfJsViewerUrl}
+                className="pdf-frame"
+                title={`PDF Page ${viewerPage}`}
+                allow="fullscreen"
+                onLoad={() => setPdfLoading(false)}
+            />
+        </div>
     );
 }

app/globals.css

@@ -228,6 +228,32 @@ h4 {
 
 /* ── PDF Viewer ─────────────────────────────────── */
 
+.pdf-container {
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  position: relative;
+  min-height: 0;
+}
+
+.pdf-loading-overlay {
+  position: absolute;
+  inset: 0;
+  z-index: 5;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  gap: 12px;
+  background: var(--pane-bg);
+  border-radius: 12px;
+}
+
+.pdf-loading-overlay p {
+  font-size: 0.85rem;
+  color: #94a3b8;
+}
+
 .pdf-frame {
   flex: 1;
   width: 100%;

app/page.js

@@ -78,11 +78,36 @@ export default function Home() {
         }
     }, []);
 
-    // Update currentDoc when selection changes
+    // Fetch annotatable pages on-demand when document selection changes
     useEffect(() => {
         if (selectedDocIndex !== null) {
             const doc = documents.find(d => d.index === selectedDocIndex);
-            setCurrentDoc(doc);
+            if (!doc) return;
+
+            // If annotatable_pages already loaded (cached), use it
+            if (doc.annotatable_pages) {
+                setCurrentDoc(doc);
+                return;
+            }
+
+            // Fetch annotatable pages on-demand
+            setLoadingPage(true);
+            fetch(`/api/document?index=${selectedDocIndex}`)
+                .then(res => res.json())
+                .then(data => {
+                    const updatedDoc = { ...doc, annotatable_pages: data.annotatable_pages || [] };
+                    setCurrentDoc(updatedDoc);
+                    // Cache in the documents array
+                    setDocuments(prev => prev.map(d =>
+                        d.index === selectedDocIndex ? updatedDoc : d
+                    ));
+                    setLoadingPage(false);
+                })
+                .catch(err => {
+                    console.error("Failed to load document pages", err);
+                    setCurrentDoc({ ...doc, annotatable_pages: [] });
+                    setLoadingPage(false);
+                });
         }
     }, [selectedDocIndex, documents]);