app.js

/**
 * FCV Data Mentions Dashboard Application Logic (Strict LLM Validation Mode)
 */

// Application State
const state = {
    allData: [],
    filteredData: [],
    selectedDocName: null,
    activeCloudWord: null, // Track currently clicked word cloud tag filter
    filters: {
        search: '',
        category: 'all',
        impactChannel: 'all',
        specificity: 'named'
    },
    pagination: {
        currentPage: 1,
        pageSize: 12
    },
    charts: {
        topMentions: null
    },
    theme: 'dark',
    activeDocPages: [],
    activePageNumber: 1,
    activeSearchTerm: "",
    activePdfUrl: "",
    docPagesCache: {} // Cache for document validated page data
};

// Helper to find the maximum page number in the active document pages
function getActiveDocMaxPage() {
    if (!state.activeDocPages || state.activeDocPages.length === 0) return 1;
    return state.activeDocPages.reduce((max, p) => p.page_number > max ? p.page_number : max, 1);
}

// Helper to normalize Unicode special characters (en-dashes, em-dashes, smart quotes) for exact PDF matches
function normalizeSearchTerm(term) {
    if (!term) return "";
    return term
        .replace(/[\u2013\u2014]/g, "-") // Convert en-dash/em-dash to regular hyphen
        .replace(/[\u2018\u2019]/g, "'") // Convert smart single quotes to straight
        .replace(/[\u201c\u201d]/g, '"') // Convert smart double quotes to straight
        .replace(/\s+/g, " ")            // Collapse multiple spaces
        .trim();
}

// Known Acronyms to keep Uppercase during normalization
const ACRONYMS = new Set(["unhcr", "wbg", "khis", "idp", "idps", "gbvims", "seis", "esmap", "step", "steps", "covid", "covid-19", "un", "who", "ngo", "cimp", "r2p", "kisedp", "khssp", "sgbv", "ipc"]);

// DOM Elements
const elements = {
    themeToggle: document.getElementById('theme-toggle'),
    globalSearch: document.getElementById('global-search'),
    filterCategory: document.getElementById('filter-category'),
    filterImpactChannel: document.getElementById('filter-impact-channel'),
    filterSpecificity: document.getElementById('filter-specificity'),
    documentList: document.getElementById('document-list'),
    filteredDocCount: document.getElementById('filtered-doc-count'),
    tabButtons: document.querySelectorAll('.tab-btn'),
    tabContents: document.querySelectorAll('.tab-content'),
    
    // KPI elements
    statTotalMentions: document.getElementById('stat-total-mentions'),
    statTotalDocs: document.getElementById('stat-total-docs'),
    statUniqueEntities: document.getElementById('stat-unique-entities'),
    cardTotalMentions: document.getElementById('card-total-mentions'),
    cardTotalDocs: document.getElementById('card-total-docs'),
    cardUniqueEntities: document.getElementById('card-unique-entities'),
    
    // Master Table elements
    tableBody: document.getElementById('table-body'),
    tableShowingCount: document.getElementById('table-showing-count'),
    tableTotalCount: document.getElementById('table-total-count'),
    paginationContainer: document.getElementById('table-pagination'),
    btnExportCsv: document.getElementById('btn-export-csv'),
    btnClearFilters: document.getElementById('btn-clear-filters'),
    
    // Word Cloud Container
    wordCloud: document.getElementById('word-cloud'),
    
    // Deep Dive elements
    diveBlankState: document.getElementById('dive-blank-state'),
    diveDetail: document.getElementById('dive-detail'),
    diveDocCategory: document.getElementById('dive-doc-category'),
    diveDocTitle: document.getElementById('dive-doc-title'),
    diveDocId: document.getElementById('dive-doc-id'),
    diveDocMentionCount: document.getElementById('dive-doc-mention-count'),
    diveMentionsList: document.getElementById('dive-mentions-list'),
    tabDocDive: document.getElementById('tab-doc-dive'),
    
    // PDF Viewer elements
    pdfViewerFrameContainer: document.getElementById('pdf-viewer-frame-container'),
    btnPrevPage: document.getElementById('btn-prev-page'),
    btnNextPage: document.getElementById('btn-next-page'),
    currentViewerPage: document.getElementById('current-viewer-page'),
    totalViewerPages: document.getElementById('total-viewer-pages')
};

// Initialize Application
document.addEventListener('DOMContentLoaded', () => {
    initTheme();
    loadData();
    setupEventListeners();
});

// Theme Management
function initTheme() {
    const savedTheme = localStorage.getItem('fcv-dashboard-theme');
    if (savedTheme) {
        state.theme = savedTheme;
    } else if (window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches) {
        state.theme = 'light';
    }
    
    document.documentElement.setAttribute('data-theme', state.theme);
}

function toggleTheme() {
    state.theme = state.theme === 'dark' ? 'light' : 'dark';
    document.documentElement.setAttribute('data-theme', state.theme);
    localStorage.setItem('fcv-dashboard-theme', state.theme);
    
    updateChartsThemes();
}

function getThemeColors() {
    const isDark = state.theme === 'dark';
    return {
        text: isDark ? '#9ca3af' : '#475569',
        border: isDark ? 'rgba(255,255,255,0.08)' : 'rgba(0,0,0,0.08)',
        grid: isDark ? 'rgba(255,255,255,0.04)' : 'rgba(0,0,0,0.04)'
    };
}

// Fetch and load Data
async function loadData() {
    try {
        const response = await fetch('data/all_mined_data_mentions.json');
        if (!response.ok) {
            throw new Error(`HTTP error! status: ${response.status}`);
        }
        const rawData = await response.json();
        
        // Clean & Normalize all records on load
        state.allData = rawData.map(item => {
            const normalized = getNormalizedMentionName(item);
            return {
                ...item,
                normalized_name: normalized
            };
        });
        
        state.filteredData = [...state.allData];
        
        // Populate filters and render UI
        populateFilters();
        applyFilters();
        initCharts();
        
        // Refresh Lucide icons
        lucide.createIcons();

        // Start background prefetching after 1.5s delay to avoid competing with initial rendering
        setTimeout(startBackgroundPrefetch, 1500);
    } catch (error) {
        console.error("Could not load dashboard data mentions:", error);
        elements.documentList.innerHTML = `
            <div class="loading-placeholder">
                <i data-lucide="alert-triangle" class="text-warning" style="width:32px;height:32px;"></i>
                <p>Failed to load dashboard data mentions. Make sure you are running a local web server (e.g. <code>python -m http.server</code>).</p>
            </div>
        `;
        lucide.createIcons();
    }
}

// Background Prefetch for JSON data and Top PDFs
async function startBackgroundPrefetch() {
    state.docPagesCache = state.docPagesCache || {};
    
    // 1. Get unique document list sorted by count (most popular first)
    const docCounts = {};
    state.allData.forEach(item => {
        if (item.document_name) {
            docCounts[item.document_name] = (docCounts[item.document_name] || 0) + 1;
        }
    });
    const sortedDocs = Object.keys(docCounts).sort((a, b) => docCounts[b] - docCounts[a]);
    
    // 2. Prefetch all JSON validation data (total ~1.5MB, very small)
    for (const docName of sortedDocs) {
        if (state.docPagesCache[docName]) continue;
        try {
            const res = await fetch(`data/validated/${docName}_validated.json`);
            if (res.ok) {
                const data = await res.json();
                state.docPagesCache[docName] = data.sort((a, b) => a.page_number - b.page_number);
            }
        } catch (e) {
            console.warn(`Background prefetch failed for JSON ${docName}:`, e);
        }
    }
    console.log("Preloaded all document text validation files.");

    // 3. Prefetch the top 5 most mentioned PDFs (~10-15MB total) to browser cache
    const topPdfs = sortedDocs.slice(0, 5);
    for (const docName of topPdfs) {
        try {
            fetch(`pdfs/${docName}.pdf`, { priority: 'low' })
                .then(res => {
                    if (res.ok) console.log(`Preloaded PDF in background: ${docName}.pdf`);
                })
                .catch(() => {});
        } catch (e) {}
    }
}

// Normalize entity mention names
function getNormalizedMentionName(item) {
    let name = item.corrected_name || item.mention_text;
    if (!name) return "Unspecified Dataset";
    
    name = name.trim();
    const words = name.split(/\s+/);
    
    const normalizedWords = words.map(w => {
        const clean = w.toLowerCase().replace(/[^a-z0-9-]/g, "");
        if (ACRONYMS.has(clean) || (w === w.toUpperCase() && w.length > 1 && isNaN(w))) {
            return w.toUpperCase();
        }
        return w.charAt(0).toUpperCase() + w.slice(1).toLowerCase();
    });
    
    return normalizedWords.join(" ");
}

// Populate Filter Dropdowns dynamically
function populateFilters() {
    const categories = new Set();
    const channels = new Set();
    
    state.allData.forEach(item => {
        if (item.corpus_category) categories.add(item.corpus_category.trim());
        if (item.downstream_impact_channel) channels.add(item.downstream_impact_channel.trim());
    });
    
    // Populate Category filter
    Array.from(categories).sort().forEach(cat => {
        const option = document.createElement('option');
        option.value = cat;
        option.textContent = cat;
        elements.filterCategory.appendChild(option);
    });
    
    // Populate Impact Channel filter
    elements.filterImpactChannel.innerHTML = '<option value="all">All Impact Channels</option>';
    Array.from(channels).sort().forEach(channel => {
        if (!channel) return;
        const option = document.createElement('option');
        option.value = channel;
        option.textContent = channel;
        elements.filterImpactChannel.appendChild(option);
    });
}

// Event Listeners Setup
function setupEventListeners() {
    elements.themeToggle.addEventListener('click', toggleTheme);
    
    // Filtering
    elements.globalSearch.addEventListener('input', (e) => {
        state.filters.search = e.target.value.toLowerCase();
        state.pagination.currentPage = 1;
        applyFilters();
    });
    
    elements.filterCategory.addEventListener('change', (e) => {
        state.filters.category = e.target.value;
        state.pagination.currentPage = 1;
        applyFilters();
    });
    
    elements.filterImpactChannel.addEventListener('change', (e) => {
        state.filters.impactChannel = e.target.value;
        state.pagination.currentPage = 1;
        applyFilters();
    });
    
    elements.filterSpecificity.addEventListener('change', (e) => {
        state.filters.specificity = e.target.value;
        state.pagination.currentPage = 1;
        applyFilters();
    });
    
    elements.btnClearFilters.addEventListener('click', resetAllFilters);
    elements.btnExportCsv.addEventListener('click', exportToCSV);
    
    elements.tabButtons.forEach(btn => {
        btn.addEventListener('click', () => {
            const tabId = btn.getAttribute('data-tab');
            switchTab(tabId);
        });
    });


    // PDF Viewer Pagination Controls
    elements.btnPrevPage.addEventListener('click', () => {
        if (state.activePageNumber > 1) {
            state.activePageNumber--;
            renderViewerContent();
            
            // Highlight corresponding mention in list if page match exists
            const pageMentions = state.allData.filter(item => item.document_name === state.selectedDocName && item.page_number === state.activePageNumber);
            if (pageMentions.length > 0) {
                state.activeSearchTerm = pageMentions[0].mention_text;
                highlightActiveMentionCard(state.activeSearchTerm);
            }
        }
    });
    
    elements.btnNextPage.addEventListener('click', () => {
        const maxPage = getActiveDocMaxPage();
        if (state.activePageNumber < maxPage) {
            state.activePageNumber++;
            renderViewerContent();
            
            // Highlight corresponding mention in list if page match exists
            const pageMentions = state.allData.filter(item => item.document_name === state.selectedDocName && item.page_number === state.activePageNumber);
            if (pageMentions.length > 0) {
                state.activeSearchTerm = pageMentions[0].mention_text;
                highlightActiveMentionCard(state.activeSearchTerm);
            }
        }
    });
}

// Reset Filters Function
function resetAllFilters() {
    state.filters.search = '';
    state.filters.category = 'all';
    state.filters.impactChannel = 'all';
    state.filters.specificity = 'named';
    state.activeCloudWord = null;
    
    elements.globalSearch.value = '';
    elements.filterCategory.value = 'all';
    elements.filterImpactChannel.value = 'all';
    elements.filterSpecificity.value = 'named';
    
    state.pagination.currentPage = 1;
    applyFilters();
}

// Apply Filters and Trigger View Renders
function applyFilters() {
    const f = state.filters;
    
    state.filteredData = state.allData.filter(item => {
        // Specificity filter
        if (f.specificity !== 'all' && item.specificity !== f.specificity) return false;
        
        // Category filter
        if (f.category !== 'all' && item.corpus_category !== f.category) return false;
        
        // Impact Channel filter
        if (f.impactChannel !== 'all' && item.downstream_impact_channel !== f.impactChannel) return false;
        
        // Word Cloud Tag filter
        if (state.activeCloudWord && item.normalized_name !== state.activeCloudWord) return false;
        
        // Search filter
        if (f.search) {
            const textToSearch = [
                item.document_name,
                item.document_title,
                item.mention_text,
                item.corrected_name,
                item.normalized_name,
                item.downstream_impact_channel,
                item.data_use_impact,
                item.context_sentence
            ].join(' ').toLowerCase();
            
            if (!textToSearch.includes(f.search)) return false;
        }
        
        return true;
    });
    
    updateStatsSummary();
    renderSidebarDocuments();
    renderMasterTable();
    renderPagination();
    renderWordCloud();
    updateCharts();
}

// Update KPI Stats Card Values
function updateStatsSummary() {
    const totalMentions = state.filteredData.length;
    
    const uniqueDocs = new Set(state.filteredData.map(item => item.document_name));
    const totalDocs = uniqueDocs.size;
    
    const uniqueEntities = new Set(state.filteredData.map(item => item.normalized_name));
    const totalUniqueEntities = uniqueEntities.size;
    
    // Bind to DOM
    elements.statTotalMentions.textContent = totalMentions;
    elements.statTotalDocs.textContent = totalDocs;
    elements.statUniqueEntities.textContent = totalUniqueEntities;
    
    elements.cardTotalMentions.textContent = totalMentions;
    elements.cardTotalDocs.textContent = totalDocs;
    elements.cardUniqueEntities.textContent = totalUniqueEntities;
}

// Render Sidebar Documents Selector
function renderSidebarDocuments() {
    const docGroups = {};
    state.filteredData.forEach(item => {
        if (!docGroups[item.document_name]) {
            docGroups[item.document_name] = {
                name: item.document_name,
                title: item.document_title || item.document_name,
                category: item.corpus_category || 'Other',
                mentionsCount: 0
            };
        }
        docGroups[item.document_name].mentionsCount++;
    });
    
    const sortedDocs = Object.values(docGroups).sort((a, b) => b.mentionsCount - a.mentionsCount);
    elements.filteredDocCount.textContent = sortedDocs.length;
    
    if (sortedDocs.length === 0) {
        elements.documentList.innerHTML = `
            <div class="loading-placeholder">
                <i data-lucide="file-warning"></i>
                <p>No matching documents found.</p>
            </div>
        `;
        lucide.createIcons();
        return;
    }
    
    elements.documentList.innerHTML = '';
    sortedDocs.forEach(doc => {
        const card = document.createElement('div');
        card.className = `doc-card ${state.selectedDocName === doc.name ? 'active' : ''}`;
        card.setAttribute('data-id', doc.name);
        
        card.innerHTML = `
            <div class="doc-card-title" title="${doc.title}">${doc.title}</div>
            <div class="doc-card-category" title="${doc.category}">${doc.category}</div>
            <div class="doc-card-footer">
                <span class="doc-card-name">${doc.name.substring(0, 18)}...</span>
                <span class="doc-card-badge">${doc.mentionsCount} mentions</span>
            </div>
        `;
        
        card.addEventListener('click', () => {
            selectDocument(doc.name);
        });
        
        elements.documentList.appendChild(card);
    });
}

// Switch tabs
function switchTab(tabId) {
    elements.tabButtons.forEach(btn => {
        if (btn.getAttribute('data-tab') === tabId) {
            btn.classList.add('active');
        } else {
            btn.classList.remove('active');
        }
    });
    
    elements.tabContents.forEach(content => {
        if (content.id === tabId) {
            content.classList.add('active');
        } else {
            content.classList.remove('active');
        }
    });
}

// Select document and reveal deep dive details
function selectDocument(docName) {
    state.selectedDocName = docName;
    
    document.querySelectorAll('.doc-card').forEach(card => {
        if (card.getAttribute('data-id') === docName) {
            card.classList.add('active');
        } else {
            card.classList.remove('active');
        }
    });
    
    switchTab('document-dive-view');
    
    const docMentions = state.allData.filter(item => item.document_name === docName);
    if (docMentions.length === 0) return;
    
    const doc = docMentions[0];
    
    elements.diveDocCategory.textContent = doc.corpus_category || 'Other Context';
    elements.diveDocTitle.textContent = doc.document_title || doc.document_name;
    elements.diveDocId.textContent = doc.document_name;
    elements.diveDocMentionCount.textContent = docMentions.length;
    
    // Set PDF state from dataset
    state.activePdfUrl = doc.pdf_url || "";
    
    elements.diveMentionsList.innerHTML = '';
    docMentions.forEach((m, idx) => {
        const hasCorrection = m.corrected_name && m.corrected_name.toLowerCase() !== m.mention_text.toLowerCase();
        const card = document.createElement('div');
        card.className = 'mention-card';
        card.setAttribute('data-mention-text', m.mention_text);
        
        card.innerHTML = `
            <div class="mention-card-header">
                <div class="mention-card-entity">${escapeHtml(m.normalized_name)}</div>
                <div class="mention-card-meta">
                    ${hasCorrection ? `<span class="badge badge-vague" style="text-decoration:line-through; font-family: monospace;">Raw: ${escapeHtml(m.mention_text)}</span>` : ''}
                    <span class="page-badge">Page ${m.page_number}</span>
                </div>
            </div>
            <div class="mention-card-body">
                <p>${highlightEntityInSentence(m.context_sentence, m.mention_text, m.corrected_name)}</p>
                ${m.data_use_impact ? `
                    <div style="font-size: 0.8rem; margin-top: 8px; color: var(--text-muted); border-top: 1px solid var(--border-color); padding-top: 8px;">
                        <i data-lucide="sparkles" style="width:14px; height:14px; margin-right:4px; display:inline-block; vertical-align:middle; color:var(--accent-primary);"></i>
                        <strong>Validated Impact:</strong> ${escapeHtml(m.data_use_impact)}
                    </div>
                ` : ''}
            </div>
            <div class="mention-card-footer">
                <span><i data-lucide="info"></i> Impact Channel: <strong>${escapeHtml(m.downstream_impact_channel || 'Unspecified')}</strong></span>
                <span><i data-lucide="tag"></i> Specificity: <strong style="text-transform: capitalize;">${escapeHtml(m.specificity || 'Unspecified')}</strong></span>
            </div>
        `;
        
        card.addEventListener('click', () => {
            state.activePageNumber = m.page_number;
            state.activeSearchTerm = m.mention_text;
            renderViewerContent();
            
            // Highlight this specific card
            document.querySelectorAll('.mention-card').forEach(c => c.classList.remove('active'));
            card.classList.add('active');
        });
        
        elements.diveMentionsList.appendChild(card);
    });
    
    // Hide blank state and display details panel
    elements.diveBlankState.classList.add('hidden');
    elements.diveDetail.classList.remove('hidden');
    
    // Initialize full text load state
    state.activeDocPages = [];
    state.activePageNumber = docMentions[0].page_number;
    state.activeSearchTerm = docMentions[0].mention_text;
    
    elements.pdfViewerFrameContainer.innerHTML = `
        <div style="display:flex; flex-direction:column; align-items:center; justify-content:center; height:100%; color:var(--text-muted); gap:12px;">
            <div class="loading-spinner"></div>
            <p>Loading document text pages...</p>
        </div>
    `;
    
    const handleLoadedPages = (data) => {
        state.activeDocPages = data.sort((a, b) => a.page_number - b.page_number);
        const maxPage = state.activeDocPages.reduce((max, p) => p.page_number > max ? p.page_number : max, 1);
        elements.totalViewerPages.textContent = maxPage;
        renderViewerContent();
        
        // Highlight the initial mention card
        setTimeout(() => {
            highlightActiveMentionCard(state.activeSearchTerm);
        }, 100);
    };

    if (state.docPagesCache && state.docPagesCache[docName]) {
        handleLoadedPages(state.docPagesCache[docName]);
    } else {
        fetch(`data/validated/${docName}_validated.json`)
            .then(response => {
                if (!response.ok) throw new Error("Document validation data not found");
                return response.json();
            })
            .then(data => {
                state.docPagesCache = state.docPagesCache || {};
                state.docPagesCache[docName] = data;
                handleLoadedPages(data);
            })
            .catch(err => {
                console.error("Failed to load document validated JSON:", err);
                elements.pdfViewerFrameContainer.innerHTML = `
                    <div style="padding:2rem; text-align:center; color:var(--text-muted); display:flex; flex-direction:column; align-items:center; justify-content:center; height:100%; gap:8px;">
                        <i data-lucide="alert-circle" style="width:36px;height:36px;color:var(--accent-primary);"></i>
                        <p>Failed to load full-text pages for this document.</p>
                    </div>
                `;
                lucide.createIcons();
            });
    }
        
    lucide.createIcons();
}

// Master Table Rendering
function renderMasterTable() {
    const { currentPage, pageSize } = state.pagination;
    const startIdx = (currentPage - 1) * pageSize;
    const endIdx = startIdx + pageSize;
    
    const paginatedItems = state.filteredData.slice(startIdx, endIdx);
    
    elements.tableShowingCount.textContent = Math.min(endIdx, state.filteredData.length);
    elements.tableTotalCount.textContent = state.filteredData.length;
    
    if (paginatedItems.length === 0) {
        elements.tableBody.innerHTML = `
            <tr>
                <td colspan="6" class="loading-placeholder">
                    <i data-lucide="search-code" style="width:32px;height:32px;opacity:0.5;"></i>
                    <p>No dataset mentions match your active filter settings.</p>
                </td>
            </tr>
        `;
        lucide.createIcons();
        return;
    }
    
    elements.tableBody.innerHTML = '';
    paginatedItems.forEach(item => {
        const tr = document.createElement('tr');
        const docTitle = item.document_title || item.document_name;
        
        tr.innerHTML = `
            <td style="font-weight:600; max-width: 150px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;" title="${escapeHtml(docTitle)}">
                <a href="#" class="table-doc-link" data-id="${item.document_name}">${escapeHtml(docTitle)}</a>
            </td>
            <td><strong>${item.page_number}</strong></td>
            <td><span style="font-weight:700; color:var(--accent-primary);">${escapeHtml(item.normalized_name)}</span></td>
            <td><code style="font-size:0.75rem; color:var(--text-muted);">${escapeHtml(item.mention_text)}</code></td>
            <td><code style="font-size:0.8rem; padding: 2px 6px; background: var(--bg-surface); border-radius: 4px;">${escapeHtml(item.downstream_impact_channel || 'unspecified')}</code></td>
            <td style="max-width: 320px; font-size: 0.8rem;">
                <div style="font-style: italic; color:var(--text-secondary);">${highlightEntityInSentence(item.context_sentence, item.mention_text, item.corrected_name)}</div>
                ${item.data_use_impact ? `
                    <div style="font-size: 0.75rem; color: var(--text-muted); margin-top: 6px; padding-top: 4px; border-top: 1px dotted var(--border-color);">
                        <i data-lucide="sparkles" style="width:12px; height:12px; margin-right:2px; display:inline-block; vertical-align:middle; color:var(--accent-primary);"></i>
                        <strong>Impact:</strong> ${escapeHtml(item.data_use_impact)}
                    </div>
                ` : ''}
            </td>
        `;
        
        tr.querySelector('.table-doc-link').addEventListener('click', (e) => {
            e.preventDefault();
            selectDocument(item.document_name);
        });
        
        elements.tableBody.appendChild(tr);
    });
    
    lucide.createIcons();
}

// Pagination rendering
function renderPagination() {
    const { currentPage, pageSize } = state.pagination;
    const totalPages = Math.ceil(state.filteredData.length / pageSize);
    
    elements.paginationContainer.innerHTML = '';
    if (totalPages <= 1) return;
    
    const prevBtn = document.createElement('button');
    prevBtn.className = 'page-btn';
    prevBtn.disabled = currentPage === 1;
    prevBtn.innerHTML = '<i data-lucide="chevron-left" style="width:14px;height:14px;"></i>';
    prevBtn.addEventListener('click', () => {
        if (state.pagination.currentPage > 1) {
            state.pagination.currentPage--;
            renderMasterTable();
            renderPagination();
        }
    });
    elements.paginationContainer.appendChild(prevBtn);
    
    const range = 2;
    let startPage = Math.max(1, currentPage - range);
    let endPage = Math.min(totalPages, currentPage + range);
    
    if (startPage > 1) {
        const pageOne = document.createElement('button');
        pageOne.className = 'page-btn';
        pageOne.textContent = '1';
        pageOne.addEventListener('click', () => {
            state.pagination.currentPage = 1;
            renderMasterTable();
            renderPagination();
        });
        elements.paginationContainer.appendChild(pageOne);
        
        if (startPage > 2) {
            const dots = document.createElement('span');
            dots.textContent = '...';
            dots.style.margin = '0 0.25rem';
            elements.paginationContainer.appendChild(dots);
        }
    }
    
    for (let i = startPage; i <= endPage; i++) {
        const btn = document.createElement('button');
        btn.className = `page-btn ${currentPage === i ? 'active' : ''}`;
        btn.textContent = i;
        btn.addEventListener('click', () => {
            state.pagination.currentPage = i;
            renderMasterTable();
            renderPagination();
        });
        elements.paginationContainer.appendChild(btn);
    }
    
    if (endPage < totalPages) {
        if (endPage < totalPages - 1) {
            const dots = document.createElement('span');
            dots.textContent = '...';
            dots.style.margin = '0 0.25rem';
            elements.paginationContainer.appendChild(dots);
        }
        
        const lastPage = document.createElement('button');
        lastPage.className = 'page-btn';
        lastPage.textContent = totalPages;
        lastPage.addEventListener('click', () => {
            state.pagination.currentPage = totalPages;
            renderMasterTable();
            renderPagination();
        });
        elements.paginationContainer.appendChild(lastPage);
    }
    
    const nextBtn = document.createElement('button');
    nextBtn.className = 'page-btn';
    nextBtn.disabled = currentPage === totalPages;
    nextBtn.innerHTML = '<i data-lucide="chevron-right" style="width:14px;height:14px;"></i>';
    nextBtn.addEventListener('click', () => {
        if (state.pagination.currentPage < totalPages) {
            state.pagination.currentPage++;
            renderMasterTable();
            renderPagination();
        }
    });
    elements.paginationContainer.appendChild(nextBtn);
    
    lucide.createIcons();
}

// Render Word Cloud
function renderWordCloud() {
    const datasource = state.activeCloudWord ? state.allData : state.filteredData;
    
    const docSets = {};
    datasource.forEach(item => {
        const name = item.normalized_name;
        if (!name) return;
        if (!docSets[name]) {
            docSets[name] = new Set();
        }
        docSets[name].add(item.document_name);
    });
    
    const freqs = {};
    Object.entries(docSets).forEach(([name, docSet]) => {
        freqs[name] = docSet.size;
    });
    
    const sortedWords = Object.entries(freqs)
        .sort((a, b) => b[1] - a[1]); // Exhaustive list
        
    if (sortedWords.length === 0) {
        elements.wordCloud.innerHTML = `
            <div class="loading-placeholder">
                <i data-lucide="cloud-off"></i>
                <p>No dataset names found to display.</p>
            </div>
        `;
        lucide.createIcons();
        return;
    }
    
    const counts = sortedWords.map(x => x[1]);
    const maxVal = Math.max(...counts);
    const minVal = Math.min(...counts);
    
    const shuffled = [...sortedWords];
    for (let i = shuffled.length - 1; i > 0; i--) {
        const j = (i * 7 + 13) % shuffled.length;
        const temp = shuffled[i];
        shuffled[i] = shuffled[j];
        shuffled[j] = temp;
    }
    
    const darkColors = ['#009FDA', '#38bdf8', '#34d399', '#60a5fa', '#a78bfa', '#f472b6', '#fb7185'];
    const lightColors = ['#0071BC', '#0284c7', '#059669', '#2563eb', '#7c3aed', '#db2777', '#e11d48'];
    const colorsList = state.theme === 'dark' ? darkColors : lightColors;
    
    elements.wordCloud.innerHTML = '';
    shuffled.forEach(([word, count]) => {
        const tag = document.createElement('span');
        tag.className = 'word-cloud-tag';
        if (state.activeCloudWord === word) {
            tag.classList.add('active-filter');
        }
        
        let size = 1.1;
        if (maxVal > minVal) {
            size = 0.85 + ((count - minVal) / (maxVal - minVal)) * (2.1 - 0.85);
        }
        
        tag.style.fontSize = `${size}rem`;
        tag.style.fontWeight = count > (maxVal + minVal)/2 ? '700' : '500';
        
        const colorIdx = word.length % colorsList.length;
        tag.style.color = colorsList[colorIdx];
        
        tag.textContent = word;
        tag.title = `${count} occurrences`;
        
        tag.addEventListener('click', () => {
            if (state.activeCloudWord === word) {
                state.activeCloudWord = null;
            } else {
                state.activeCloudWord = word;
            }
            state.pagination.currentPage = 1;
            applyFilters();
        });
        
        elements.wordCloud.appendChild(tag);
    });
}

// Chart.js Visualizations Setup
function initCharts() {
    const mentionsCanvas = document.getElementById('chart-top-mentions');
    if (!mentionsCanvas) return;
    
    const tc = getThemeColors();
    
    state.charts.topMentions = new Chart(mentionsCanvas, {
        type: 'bar',
        data: {
            labels: [],
            datasets: [{
                label: 'Documents',
                data: [],
                backgroundColor: 'rgba(0, 113, 188, 0.7)',
                borderColor: '#0071BC',
                borderWidth: 1,
                borderRadius: 4
            }]
        },
        options: {
            indexAxis: 'y',
            responsive: true,
            maintainAspectRatio: false,
            plugins: {
                legend: { display: false }
            },
            scales: {
                x: {
                    ticks: { color: tc.text, font: { family: 'Inter', size: 10 } },
                    grid: { color: tc.grid }
                },
                y: {
                    ticks: { color: tc.text, font: { family: 'Inter', size: 10 } },
                    grid: { display: false }
                }
            },
            onClick: (event, elements) => {
                if (elements.length > 0) {
                    const idx = elements[0].index;
                    const label = state.charts.topMentions.data.labels[idx];
                    
                    if (state.activeCloudWord === label) {
                        state.activeCloudWord = null;
                    } else {
                        state.activeCloudWord = label;
                    }
                    state.pagination.currentPage = 1;
                    applyFilters();
                }
            }
        }
    });
    
    updateCharts();
}

// Update charts themes dynamically (Gridlines/Colors)
function updateChartsThemes() {
    const tc = getThemeColors();
    const chart = state.charts.topMentions;
    
    if (chart) {
        if (chart.options.scales) {
            if (chart.options.scales.x) {
                chart.options.scales.x.ticks.color = tc.text;
                chart.options.scales.x.grid.color = tc.grid;
            }
            if (chart.options.scales.y) {
                chart.options.scales.y.ticks.color = tc.text;
                chart.options.scales.y.grid.color = tc.grid;
            }
        }
        chart.update();
    }
    
    renderWordCloud();
}

// Re-aggregate and update charts datasets based on filters
function updateCharts() {
    const chart = state.charts.topMentions;
    if (!chart) return;
    
    const docSets = {};
    state.filteredData.forEach(item => {
        const name = item.normalized_name;
        if (!name) return;
        if (!docSets[name]) {
            docSets[name] = new Set();
        }
        docSets[name].add(item.document_name);
    });
    
    const counts = {};
    Object.entries(docSets).forEach(([name, docSet]) => {
        counts[name] = docSet.size;
    });
    
    const sorted = Object.entries(counts)
        .sort((a, b) => b[1] - a[1])
        .slice(0, 15);
        
    chart.data.labels = sorted.map(x => x[0]);
    chart.data.datasets[0].data = sorted.map(x => x[1]);
    chart.update();
}

// Highlight entity or corrected entity inside its context sentence
function highlightEntityInSentence(sentence, mention, corrected) {
    if (!sentence) return '';
    
    const escapeRegexStr = (str) => str.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
    
    const termsToTry = [];
    if (corrected && corrected.trim()) termsToTry.push(corrected.trim());
    if (mention && mention.trim()) termsToTry.push(mention.trim());
    
    let highlighted = escapeHtml(sentence);
    
    for (const term of termsToTry) {
        if (!term) continue;
        const cleanTerm = escapeRegexStr(term);
        const matchRegex = new RegExp(`(${cleanTerm})`, 'gi');
        
        const escapedTerm = escapeHtml(term);
        const escapedMatchRegex = new RegExp(`(${escapeRegexStr(escapedTerm)})`, 'gi');
        
        if (escapedMatchRegex.test(highlighted)) {
            highlighted = highlighted.replace(escapedMatchRegex, '<mark>$1</mark>');
            return highlighted;
        }
    }
    
    if (mention) {
        const firstWord = mention.split(/\s+/)[0];
        if (firstWord && firstWord.length > 2) {
            const escFirstWord = escapeHtml(firstWord);
            const regex = new RegExp(`(${escapeRegexStr(escFirstWord)})`, 'gi');
            if (regex.test(highlighted)) {
                return highlighted.replace(regex, '<mark>$1</mark>');
            }
        }
    }
    
    return highlighted;
}

// Helper to escape HTML tags to prevent XSS
function escapeHtml(unsafe) {
    if (!unsafe) return '';
    return unsafe
        .replace(/&/g, "&amp;")
        .replace(/</g, "&lt;")
        .replace(/>/g, "&gt;")
        .replace(/"/g, "&quot;")
        .replace(/'/g, "&#039;");
}

// Export Filtered Mentions to CSV File
function exportToCSV() {
    if (state.filteredData.length === 0) {
        alert("There is no data to export.");
        return;
    }
    
    const headers = [
        "document_name", "document_title", "corpus_category", "page_number", 
        "mention_text", "corrected_name", "normalized_name",
        "downstream_impact_channel", "data_use_impact", "context_sentence"
    ];
    
    const csvRows = [headers.join(",")];
    
    state.filteredData.forEach(item => {
        const values = headers.map(header => {
            const val = item[header] === undefined || item[header] === null ? '' : String(item[header]);
            const escaped = val.replace(/"/g, '""');
            return `"${escaped}"`;
        });
        csvRows.push(values.join(","));
    });
    
    const csvContent = "\ufeff" + csvRows.join("\n");
    const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' });
    
    const link = document.createElement("a");
    if (link.download !== undefined) {
        const url = URL.createObjectURL(blob);
        link.setAttribute("href", url);
        link.setAttribute("download", `fcv_data_mentions_export_${Date.now()}.csv`);
        link.style.visibility = 'hidden';
        document.body.appendChild(link);
        link.click();
        document.body.removeChild(link);
    }
}

// Render the split-pane Viewer (PDF.js iframe OR dynamic Markdown text layer)
function renderViewerContent() {
    if (state.activeDocPages.length === 0) return;
    
    // Update pagination controls
    const maxPage = getActiveDocMaxPage();
    elements.currentViewerPage.textContent = state.activePageNumber;
    elements.btnPrevPage.disabled = state.activePageNumber <= 1;
    elements.btnNextPage.disabled = state.activePageNumber >= maxPage;
    
    if (state.activePdfUrl) {
        // PDF is available: Load Mozilla PDF.js
        let pdfFileUrl = state.activePdfUrl;
        if (pdfFileUrl && !pdfFileUrl.startsWith('/') && !pdfFileUrl.startsWith('http')) {
            pdfFileUrl = '/' + pdfFileUrl;
        }
        
        const cleanSearch = normalizeSearchTerm(state.activeSearchTerm);
        const hash = `#page=${state.activePageNumber}${cleanSearch ? `&search=${encodeURIComponent(cleanSearch)}&phrase=true` : ''}`;
        const viewerUrl = `pdfjs/web/viewer.html?file=${encodeURIComponent(pdfFileUrl)}${hash}`;
        
        const existingIframe = document.getElementById('pdf-viewer');
        if (existingIframe && existingIframe.getAttribute('data-pdf-url') === pdfFileUrl) {
            // Document already loaded: update hash to jump to page and highlight instantly
            try {
                existingIframe.contentWindow.location.hash = hash;
            } catch (e) {
                existingIframe.src = viewerUrl;
            }
        } else {
            // Load new document: recreate iframe
            elements.pdfViewerFrameContainer.innerHTML = `
                <iframe 
                    id="pdf-viewer"
                    data-pdf-url="${pdfFileUrl}"
                    src="${viewerUrl}"
                    class="pdf-frame"
                    title="PDF Viewer Pane"
                    allow="fullscreen"
                ></iframe>
            `;
        }
    } else {
        // PDF is NOT available: Render high-fidelity text fallback with mentions highlighted
        const pageData = state.activeDocPages.find(p => p.page_number === state.activePageNumber);
        if (!pageData) {
            elements.pdfViewerFrameContainer.innerHTML = `<div class="fallback-text-viewer">Page data not found for Page ${state.activePageNumber}.</div>`;
            return;
        }
        
        let text = pageData.input_text_preprocessed || pageData.input_text || "";
        
        // Find all validated mentions in this document on the current page to highlight them
        const docMentions = state.allData.filter(item => item.document_name === state.selectedDocName && item.page_number === state.activePageNumber);
        
        // Sort by length desc to avoid inner-substring matching issues
        const sortedMentions = [...docMentions].sort((a, b) => b.mention_text.length - a.mention_text.length);
        
        let escapedText = escapeHtml(text);
        
        // Replace matches with temporary token tags to avoid double-processing
        const replacements = [];
        sortedMentions.forEach((m, idx) => {
            const term = m.mention_text;
            const isCurrent = term.toLowerCase() === state.activeSearchTerm.toLowerCase();
            const placeholder = `___MENTION_TOKEN_VAL_${idx}___`;
            
            const escapedTerm = escapeHtml(term);
            const regex = new RegExp(escapeRegExp(escapedTerm), 'gi');
            
            escapedText = escapedText.replace(regex, (match) => {
                replacements.push({
                    placeholder: placeholder,
                    replacement: `<mark class="${isCurrent ? 'selected-highlight' : 'other-mention-highlight'}" onclick="window.setActiveSearchTerm('${escapeJsString(term)}')">${match}</mark>`
                });
                return placeholder;
            });
        });
        
        // Restore replaced placeholders
        replacements.forEach(rep => {
            escapedText = escapedText.replace(rep.placeholder, rep.replacement);
        });
        
        elements.pdfViewerFrameContainer.innerHTML = `
            <div class="fallback-text-viewer" id="fallback-text-viewer">
                ${escapedText}
            </div>
        `;
        
        // Scroll the selected highlighted mention into view
        setTimeout(() => {
            const selectedMark = document.querySelector('.selected-highlight');
            if (selectedMark) {
                selectedMark.scrollIntoView({ behavior: 'smooth', block: 'center' });
            }
        }, 80);
    }
}

// Highlight the active mention card in the scrollable inventory pane
function highlightActiveMentionCard(term) {
    document.querySelectorAll('.mention-card').forEach(card => {
        if (card.getAttribute('data-mention-text') === term) {
            card.classList.add('active');
            card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
        } else {
            card.classList.remove('active');
        }
    });
}

// Globally register the highlight clicks inside the fallback Markdown reader
window.setActiveSearchTerm = function(term) {
    state.activeSearchTerm = term;
    renderViewerContent();
    highlightActiveMentionCard(term);
};

// Escape regex special characters
function escapeRegExp(string) {
    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

// Escape javascript string single/double quotes
function escapeJsString(str) {
    return str.replace(/'/g, "\\'").replace(/"/g, '\\"');
}