const getApiBase = () => { if (localStorage.getItem("LLMBENCH_BACKEND_URL")) { return localStorage.getItem("LLMBENCH_BACKEND_URL"); } if (window.location.hostname.includes("github.io")) { return "https://jimmy2110-llmbench.hf.space/api/v1"; } return "/api/v1"; }; const API_BASE = getApiBase(); const API_KEY = 'evalforge_admin_secret_key'; const HEADERS = { 'X-API-Key': API_KEY, 'Content-Type': 'application/json' }; // Global Store State let globalRuns = []; let globalDatasets = []; let globalPrompts = []; let activeTab = 'dashboard'; // --- Lifecycle Event Handlers --- document.addEventListener("DOMContentLoaded", () => { initNavigation(); loadAllData(); initScrollReveal(); // Bind forms document.getElementById("create-dataset-form").addEventListener("submit", handleCreateDataset); document.getElementById("add-single-case-form").addEventListener("submit", handleAddSingleTestCase); document.getElementById("create-prompt-form").addEventListener("submit", handleRegisterPrompt); document.getElementById("trigger-run-form").addEventListener("submit", handleTriggerEvaluation); }); // Load all datasets at boot for dynamic scrolling page layout async function loadAllData() { try { await Promise.all([ loadDashboardData(), loadDatasetsList(), loadArenaPrompts(), loadHubConfiguration(), loadRcaConfiguration(), loadCostAnalytics() ]); } catch (err) { console.error("Error loading dashboard data:", err); } } // --- SPA Scrolling Navigation Manager --- function initNavigation() { const navLinks = document.querySelectorAll(".menu-item"); const sections = document.querySelectorAll(".scroll-section"); // Scroll progress line tracker window.addEventListener("scroll", () => { const winScroll = document.documentElement.scrollTop || document.body.scrollTop; const height = document.documentElement.scrollHeight - document.documentElement.clientHeight; const scrolled = height > 0 ? (winScroll / height) * 100 : 0; const progressBar = document.getElementById("scroll-progress"); if (progressBar) progressBar.style.width = scrolled + "%"; }); // Intersection observer for section activations & entrance animations const observerOptions = { root: null, threshold: 0.08, rootMargin: "-10% 0px -40% 0px" }; const observer = new IntersectionObserver((entries) => { entries.forEach(entry => { if (entry.isIntersecting) { entry.target.classList.add("visible"); const sectionId = entry.target.id; // Highlight active nav item navLinks.forEach(link => { const href = link.getAttribute("href"); if (href === `#${sectionId}`) { link.classList.add("active"); } else { link.classList.remove("active"); } }); } }); }, observerOptions); sections.forEach(section => { observer.observe(section); }); // Smooth navigation clicking navLinks.forEach(link => { link.addEventListener("click", (e) => { e.preventDefault(); const targetId = link.getAttribute("href"); const targetSection = document.querySelector(targetId); if (targetSection) { const headerOffset = 90; const elementPosition = targetSection.getBoundingClientRect().top; const offsetPosition = elementPosition + window.pageYOffset - headerOffset; window.scrollTo({ top: offsetPosition, behavior: "smooth" }); // Set history hash history.pushState(null, null, targetId); } }); }); // Check URL hash on page mount to scroll there if (window.location.hash) { const hash = window.location.hash; const targetSection = document.querySelector(hash); if (targetSection) { setTimeout(() => { const headerOffset = 90; const elementPosition = targetSection.getBoundingClientRect().top; const offsetPosition = elementPosition + window.pageYOffset - headerOffset; window.scrollTo({ top: offsetPosition, behavior: "auto" }); }, 300); } } } // Inner tab selectors function switchInnerTab(tabName, btn) { const parentContainer = btn.closest(".inner-tabs-container"); // Deactivate sibling tabs parentContainer.querySelectorAll(".inner-tab-btn").forEach(item => { item.classList.remove("active"); }); btn.classList.add("active"); // Deactivate panels parentContainer.querySelectorAll(".inner-tab-panel").forEach(panel => { if (panel.id === `inner-tab-${tabName}`) { panel.classList.add("active"); } else { panel.classList.remove("active"); } }); } function toggleExpander(expanderId) { const expander = document.getElementById(expanderId); const header = expander.previousElementSibling; const icon = header.querySelector(".expander-icon"); if (expander.classList.contains("hidden")) { expander.classList.remove("hidden"); icon.textContent = "-"; } else { expander.classList.add("hidden"); icon.textContent = "+"; } } // --- API Helpers --- async function fetchApi(endpoint, options = {}) { try { const response = await fetch(`${API_BASE}${endpoint}`, { headers: HEADERS, ...options }); if (!response.ok) { const errText = await response.text(); throw new Error(errText || response.statusText); } return await response.json(); } catch (err) { console.error(`API Fetch Error [${endpoint}]:`, err); throw err; } } // --- 1. Overview Dashboard Operations --- async function loadDashboardData() { try { // Fetch raw metrics from backend simultaneously const [datasets, prompts, runs] = await Promise.all([ fetchApi('/datasets/'), fetchApi('/prompts/'), fetchApi('/evaluations/') ]); globalDatasets = datasets; globalPrompts = prompts; globalRuns = runs; // Count totals document.getElementById("kpi-datasets").textContent = datasets.length; document.getElementById("kpi-prompts").textContent = prompts.length; document.getElementById("kpi-runs").textContent = runs.length; // Process cost metric const completedRuns = runs.filter(r => r.status === 'COMPLETED'); const totalSpent = completedRuns.reduce((acc, r) => acc + (r.metrics?.total_cost || 0), 0); document.getElementById("kpi-cost").textContent = `$${totalSpent.toFixed(4)}`; // Populate recent runs table populateRecentRunsTable(runs); // Draw bubble chart renderFrontierChart(completedRuns); } catch (err) { console.error("Failed to load dashboard statistics:", err); } } function populateRecentRunsTable(runs) { const tbody = document.getElementById("recent-runs-table-body"); tbody.innerHTML = ""; if (runs.length === 0) { tbody.innerHTML = `No evaluation runs recorded. Run a benchmark first.`; return; } // Limit to latest 5 runs runs.slice(0, 5).forEach(run => { const tr = document.createElement("tr"); const dt = new Date(run.created_at); const dateStr = run.created_at ? dt.toLocaleString() : 'N/A'; const costStr = run.metrics?.total_cost ? `$${run.metrics.total_cost.toFixed(4)}` : 'N/A'; const accStr = run.metrics?.average_accuracy ? run.metrics.average_accuracy.toFixed(2) : 'N/A'; const hallStr = run.metrics?.hallucination_rate !== undefined ? `${(run.metrics.hallucination_rate * 100).toFixed(1)}%` : 'N/A'; tr.innerHTML = ` ${run.id} ${run.dataset_id} ${run.prompt_id || 'None'} ${run.status} ${accStr} ${hallStr} ${costStr} ${dateStr} `; tbody.appendChild(tr); }); } function renderFrontierChart(completedRuns) { // Generate averages grouped by model const modelMetrics = {}; completedRuns.forEach(run => { const results = run.results || []; results.forEach(res => { const m = res.model_name; if (!modelMetrics[m]) { modelMetrics[m] = { accuracy: [], cost: [], latency: [] }; } if (res.accuracy !== null) modelMetrics[m].accuracy.push(res.accuracy); modelMetrics[m].cost.push(res.cost || 0.0); modelMetrics[m].latency.push(res.latency_ms || 0.0); }); }); const dataRows = []; Object.keys(modelMetrics).forEach(model => { const accs = modelMetrics[model].accuracy; const costs = modelMetrics[model].cost; const lats = modelMetrics[model].latency; const avgAcc = accs.length ? accs.reduce((a,b)=>a+b,0)/accs.length : 0.0; const avgCost = costs.length ? costs.reduce((a,b)=>a+b,0)/costs.length : 0.0; const avgLat = lats.length ? lats.reduce((a,b)=>a+b,0)/lats.length : 0.0; dataRows.push({ model, accuracy: avgAcc, cost: avgCost, latency: avgLat }); }); // Fallbacks if database has no items yet if (dataRows.length === 0) { dataRows.push( { model: "gpt-4o", accuracy: 4.75, cost: 0.0125, latency: 1250.0 }, { model: "claude-3-5-sonnet", accuracy: 4.82, cost: 0.0095, latency: 1500.0 }, { model: "gemini-1.5-flash", accuracy: 4.10, cost: 0.0003, latency: 650.0 }, { model: "gemini-1.5-pro", accuracy: 4.60, cost: 0.0045, latency: 1900.0 } ); } const traces = dataRows.map(row => ({ x: [row.cost], y: [row.accuracy], mode: 'markers+text', name: row.model, text: [row.model], textposition: 'top center', marker: { size: [Math.max(15, row.latency / 50)], sizeref: 1, sizemode: 'area', color: getRandomColor(row.model), line: { width: 1, color: '#2E221E' } }, hovertemplate: `%{text}
Accuracy: %{y:.2f}/5.0
Cost: $%{x:.6f}
Latency: ${row.latency.toFixed(0)} ms` })); const layout = { xaxis: { title: 'Average Cost (USD log scale)', type: 'log', gridcolor: 'rgba(194, 125, 56, 0.08)', linecolor: 'rgba(194, 125, 56, 0.15)', zerolinecolor: 'rgba(194, 125, 56, 0.15)', tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, titlefont: { color: '#C27D38', family: 'Playfair Display, serif' } }, yaxis: { title: 'Average Accuracy (1-5)', range: [3.5, 5.0], gridcolor: 'rgba(194, 125, 56, 0.08)', linecolor: 'rgba(194, 125, 56, 0.15)', zerolinecolor: 'rgba(194, 125, 56, 0.15)', tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, titlefont: { color: '#C27D38', family: 'Playfair Display, serif' } }, paper_bgcolor: 'rgba(0, 0, 0, 0)', plot_bgcolor: 'rgba(0, 0, 0, 0)', font: { color: '#2E221E', family: 'Plus Jakarta Sans, sans-serif' }, showlegend: true, legend: { font: { color: '#2E221E' } }, margin: { t: 40, b: 60, l: 60, r: 40 } }; Plotly.newPlot('frontier-chart', traces, layout); } function getRandomColor(model) { // Return standard thematic colors for primary models, otherwise random green-mint shades if (model.includes("gpt-4o")) return '#3b82f6'; // Blue if (model.includes("claude")) return '#ea580c'; // Orange if (model.includes("gemini-1.5-flash")) return '#eab308'; // Yellow if (model.includes("gemini-1.5-pro")) return '#10b981'; // Emerald Green return '#D2A275'; // Sage Green default } // --- 2. Dataset Management Operations --- async function loadDatasetsList() { try { const datasets = await fetchApi('/datasets/'); globalDatasets = datasets; const selector = document.getElementById("ds-selector"); selector.innerHTML = ``; datasets.forEach(ds => { const opt = document.createElement("option"); opt.value = ds.id; opt.textContent = `${ds.name} (v${ds.version}) - ID: ${ds.id}`; selector.appendChild(opt); }); // Hide details container until loaded document.getElementById("dataset-details-container").classList.add("hidden"); } catch (err) { alert("Failed to query datasets list from backend API."); } } async function handleCreateDataset(e) { e.preventDefault(); const payload = { name: document.getElementById("ds-name").value, version: document.getElementById("ds-version").value, category: document.getElementById("ds-category").value, description: document.getElementById("ds-description").value }; try { const newDs = await fetchApi('/datasets/', { method: 'POST', body: JSON.stringify(payload) }); alert(`Dataset "${newDs.name}" successfully created!`); document.getElementById("create-dataset-form").reset(); toggleExpander('create-dataset-expander'); loadDatasetsList(); } catch (err) { alert(`Failed to save dataset: ${err.message}`); } } async function loadSelectedDataset() { const dsId = document.getElementById("ds-selector").value; if (!dsId) { document.getElementById("dataset-details-container").classList.add("hidden"); return; } const ds = globalDatasets.find(d => d.id == dsId); if (!ds) return; // Show panel details document.getElementById("dataset-details-container").classList.remove("hidden"); document.getElementById("inspect-ds-title").textContent = ds.name; document.getElementById("inspect-ds-version").textContent = ds.version; document.getElementById("inspect-ds-category").textContent = ds.category; document.getElementById("inspect-ds-description").textContent = ds.description || "No description provided."; // Pre-populate input defaults document.getElementById("tc-category").value = ds.category; // Query test cases list await loadTestCasesList(dsId); } async function loadTestCasesList(dsId) { try { const cases = await fetchApi(`/datasets/${dsId}/testcases`); const tbody = document.getElementById("testcases-table-body"); const deleteSelector = document.getElementById("delete-tc-selector"); tbody.innerHTML = ""; deleteSelector.innerHTML = ``; if (cases.length === 0) { tbody.innerHTML = `No test cases registered for this dataset yet.`; return; } cases.forEach(tc => { const tr = document.createElement("tr"); tr.innerHTML = ` ${tc.id} ${tc.category} ${escapeHtml(tc.question)} ${escapeHtml(tc.ground_truth)} `; tbody.appendChild(tr); const opt = document.createElement("option"); opt.value = tc.id; opt.textContent = `TestCase ID: ${tc.id}`; deleteSelector.appendChild(opt); }); } catch (err) { console.error("Error fetching test cases list:", err); } } async function directDeleteTestCase(dsId, tcId) { if (!confirm(`Are you sure you want to delete Test Case ID ${tcId}?`)) return; try { await fetchApi(`/datasets/${dsId}/testcases/${tcId}`, { method: 'DELETE' }); alert(`Test Case ${tcId} deleted successfully.`); loadTestCasesList(dsId); } catch (err) { alert("Failed to delete test case."); } } async function deleteTestCase() { const dsId = document.getElementById("ds-selector").value; const tcId = document.getElementById("delete-tc-selector").value; if (!tcId) { alert("Please select a valid Test Case ID."); return; } await directDeleteTestCase(dsId, tcId); } async function uploadBatchTestCases() { const dsId = document.getElementById("ds-selector").value; const jsonText = document.getElementById("upload-json-input").value; if (!jsonText.trim()) { alert("Please input a valid JSON array."); return; } try { const payload = JSON.parse(jsonText); if (!Array.isArray(payload)) { alert("The pasted content must be a JSON array (wrapped in [])."); return; } const res = await fetchApi(`/datasets/${dsId}/testcases/batch`, { method: 'POST', body: JSON.stringify(payload) }); alert(`Success! ${res.message || 'Batch uploaded successfully.'}`); document.getElementById("upload-json-input").value = ""; loadTestCasesList(dsId); } catch (err) { alert(`Failed to upload batch. Verify syntax is valid JSON. Details: ${err.message}`); } } async function handleAddSingleTestCase(e) { e.preventDefault(); const dsId = document.getElementById("ds-selector").value; let meta = {}; try { const metaStr = document.getElementById("tc-metadata").value; if (metaStr.trim()) meta = JSON.parse(metaStr); } catch (err) { alert("Invalid metadata JSON object format."); return; } const payload = { question: document.getElementById("tc-question").value, ground_truth: document.getElementById("tc-ground-truth").value, category: document.getElementById("tc-category").value, meta_data: meta }; try { await fetchApi(`/datasets/${dsId}/testcases`, { method: 'POST', body: JSON.stringify(payload) }); alert("Single test case appended successfully!"); document.getElementById("add-single-case-form").reset(); document.getElementById("tc-category").value = globalDatasets.find(d => d.id == dsId).category; loadTestCasesList(dsId); } catch (err) { alert(`Failed to save test case: ${err.message}`); } } // --- 3. Prompt Arena & Benchmark Operations --- async function loadArenaPrompts() { try { const prompts = await fetchApi('/prompts/'); globalPrompts = prompts; const groupSelector = document.getElementById("arena-group-selector"); groupSelector.innerHTML = ``; const uniqueNames = [...new Set(prompts.map(p => p.name))]; uniqueNames.forEach(name => { const opt = document.createElement("option"); opt.value = name; opt.textContent = name; groupSelector.appendChild(opt); }); document.getElementById("arena-comparison-sidebyside").classList.add("hidden"); document.getElementById("arena-regression-panel").classList.add("hidden"); } catch (err) { console.error("Failed to query prompts list:", err); } } async function handleRegisterPrompt(e) { e.preventDefault(); const payload = { name: document.getElementById("pr-name").value, version: document.getElementById("pr-version").value, system_prompt: document.getElementById("pr-system").value || null, user_template: document.getElementById("pr-user").value, description: document.getElementById("pr-description").value || null }; try { const newPr = await fetchApi('/prompts/', { method: 'POST', body: JSON.stringify(payload) }); alert(`Prompt template "${newPr.name}" version "${newPr.version}" successfully registered!`); document.getElementById("create-prompt-form").reset(); toggleExpander('create-prompt-expander'); loadArenaPrompts(); } catch (err) { alert(`Failed to register prompt: ${err.message}`); } } function loadArenaGroupVersions() { const groupName = document.getElementById("arena-group-selector").value; const baseSelector = document.getElementById("arena-base-selector"); const compSelector = document.getElementById("arena-comp-selector"); baseSelector.innerHTML = ``; compSelector.innerHTML = ``; if (!groupName) { document.getElementById("arena-comparison-sidebyside").classList.add("hidden"); document.getElementById("arena-regression-panel").classList.add("hidden"); return; } const groupPrompts = globalPrompts.filter(p => p.name === groupName); groupPrompts.forEach(p => { const optA = document.createElement("option"); optA.value = p.id; optA.textContent = p.version; baseSelector.appendChild(optA); const optB = document.createElement("option"); optB.value = p.id; optB.textContent = p.version; compSelector.appendChild(optB); }); } function renderArenaComparison() { const baseId = document.getElementById("arena-base-selector").value; const compId = document.getElementById("arena-comp-selector").value; if (!baseId || !compId) { document.getElementById("arena-comparison-sidebyside").classList.add("hidden"); document.getElementById("arena-regression-panel").classList.add("hidden"); return; } const basePrompt = globalPrompts.find(p => p.id == baseId); const compPrompt = globalPrompts.find(p => p.id == compId); if (basePrompt && compPrompt) { document.getElementById("arena-comparison-sidebyside").classList.remove("hidden"); document.getElementById("arena-base-system").textContent = basePrompt.system_prompt || 'None'; document.getElementById("arena-base-user").textContent = basePrompt.user_template; document.getElementById("arena-comp-system").textContent = compPrompt.system_prompt || 'None'; document.getElementById("arena-comp-user").textContent = compPrompt.user_template; // Query completed runs list to select and benchmark them loadArenaRegressionRuns(baseId, compId); } } async function loadArenaRegressionRuns(basePromptId, compPromptId) { try { const runs = await fetchApi('/evaluations/'); const completedRuns = runs.filter(r => r.status === 'COMPLETED'); const baseRunSelector = document.getElementById("arena-base-run-selector"); const compRunSelector = document.getElementById("arena-comp-run-selector"); baseRunSelector.innerHTML = ``; compRunSelector.innerHTML = ``; const basePromptRuns = completedRuns.filter(r => r.prompt_id == basePromptId); const compPromptRuns = completedRuns.filter(r => r.prompt_id == compPromptId); if (basePromptRuns.length === 0 || compPromptRuns.length === 0) { document.getElementById("arena-regression-panel").classList.remove("hidden"); document.getElementById("regression-report-container").classList.add("hidden"); baseRunSelector.innerHTML = ``; compRunSelector.innerHTML = ``; return; } basePromptRuns.forEach(r => { const opt = document.createElement("option"); opt.value = r.id; opt.textContent = `Run ID: ${r.id} (Dataset ID: ${r.dataset_id}) - Accuracy: ${r.metrics?.average_accuracy?.toFixed(2) || '0.00'}`; baseRunSelector.appendChild(opt); }); compPromptRuns.forEach(r => { const opt = document.createElement("option"); opt.value = r.id; opt.textContent = `Run ID: ${r.id} (Dataset ID: ${r.dataset_id}) - Accuracy: ${r.metrics?.average_accuracy?.toFixed(2) || '0.00'}`; compRunSelector.appendChild(opt); }); document.getElementById("arena-regression-panel").classList.remove("hidden"); document.getElementById("regression-report-container").classList.add("hidden"); } catch (err) { console.error("Failed to load prompt benchmark runs:", err); } } async function triggerRegressionReport() { const baseRunId = document.getElementById("arena-base-run-selector").value; const compRunId = document.getElementById("arena-comp-run-selector").value; if (!baseRunId || !compRunId) { alert("Please select both baseline and comparison runs to analyze."); return; } try { const payload = { baseline_run_id: parseInt(baseRunId), comparison_run_id: parseInt(compRunId) }; const report = await fetchApi('/evaluations/compare', { method: 'POST', body: JSON.stringify(payload) }); const findings = report.findings || {}; // Show report details container document.getElementById("regression-report-container").classList.remove("hidden"); // Accuracy Delta Card const accDelta = report.score_delta; const accEl = document.getElementById("reg-kpi-accuracy"); accEl.textContent = `${accDelta >= 0 ? '+' : ''}${accDelta.toFixed(2)}`; accEl.style.color = accDelta >= 0 ? 'var(--accent-green)' : 'var(--accent-red)'; document.getElementById("reg-sub-accuracy").textContent = `Baseline: ${findings.baseline_accuracy?.toFixed(2) || '0.00'}`; // Latency Delta Card const latDelta = findings.latency_delta_ms; const latEl = document.getElementById("reg-kpi-latency"); latEl.textContent = `${latDelta >= 0 ? '+' : ''}${latDelta.toFixed(1)} ms`; latEl.style.color = latDelta <= 0 ? 'var(--accent-green)' : 'var(--accent-red)'; // Cost Delta Card const costDelta = findings.cost_delta; const costEl = document.getElementById("reg-kpi-cost"); costEl.textContent = `${costDelta >= 0 ? '+' : ''}$${costDelta.toFixed(5)}`; costEl.style.color = costDelta <= 0 ? 'var(--accent-green)' : 'var(--accent-red)'; // Hallucination Delta Card const hallDelta = findings.hallucination_rate_delta * 100; const hallEl = document.getElementById("reg-kpi-hallucination"); hallEl.textContent = `${hallDelta >= 0 ? '+' : ''}${hallDelta.toFixed(2)}%`; hallEl.style.color = hallDelta <= 0 ? 'var(--accent-green)' : 'var(--accent-red)'; // Render comparative radar plot renderRegressionRadarPlot(findings, baseRunId, compRunId); } catch (err) { alert("Failed to analyze runs regression delta."); } } async function renderRegressionRadarPlot(findings, baseRunId, compRunId) { try { const [baseRun, compRun] = await Promise.all([ fetchApi(`/evaluations/${baseRunId}`), fetchApi(`/evaluations/${compRunId}`) ]); const mBase = baseRun.metrics || {}; const mComp = compRun.metrics || {}; const categories = ['Accuracy', 'Completeness', 'Hallucination Resistance', 'Tone', 'Reasoning']; const baseScores = [ mBase.average_accuracy || 0, mBase.average_completeness || 0, mBase.average_hallucination || 0, mBase.average_tone || 0, mBase.average_reasoning || 0 ]; const compScores = [ mComp.average_accuracy || 0, mComp.average_completeness || 0, mComp.average_hallucination || 0, mComp.average_tone || 0, mComp.average_reasoning || 0 ]; const traces = [ { type: 'scatterpolar', r: baseScores, theta: categories, fill: 'toself', name: `Baseline (A) - Run ${baseRunId}`, line: { color: '#C27D38' }, fillcolor: 'rgba(194, 125, 56, 0.15)' }, { type: 'scatterpolar', r: compScores, theta: categories, fill: 'toself', name: `Comparison (B) - Run ${compRunId}`, line: { color: '#A64B2A' }, fillcolor: 'rgba(166, 75, 42, 0.25)' } ]; const layout = { polar: { radialaxis: { visible: true, range: [0, 5], color: '#C27D38', gridcolor: 'rgba(194, 125, 56, 0.12)' }, angularaxis: { gridcolor: 'rgba(194, 125, 56, 0.12)', linecolor: '#C27D38' }, bgcolor: 'rgba(0, 0, 0, 0)' }, showlegend: true, paper_bgcolor: 'rgba(0, 0, 0, 0)', plot_bgcolor: 'rgba(0, 0, 0, 0)', font: { color: '#2E221E', family: 'Plus Jakarta Sans, sans-serif' }, margin: { t: 40, b: 40, l: 40, r: 40 } }; Plotly.newPlot('arena-radar-chart', traces, layout); } catch (err) { console.error("Failed to load run details for radar chart comparative analysis:", err); } } // --- 4. Evaluation Hub Operations --- async function loadHubConfiguration() { try { const [datasets, prompts, runs] = await Promise.all([ fetchApi('/datasets/'), fetchApi('/prompts/'), fetchApi('/evaluations/') ]); globalDatasets = datasets; globalPrompts = prompts; globalRuns = runs; // Populate datasets select const dsSelect = document.getElementById("hub-ds-selector"); dsSelect.innerHTML = ``; datasets.forEach(ds => { const opt = document.createElement("option"); opt.value = ds.id; opt.textContent = `${ds.name} (v${ds.version}) - ID: ${ds.id}`; dsSelect.appendChild(opt); }); // Populate prompts select const prSelect = document.getElementById("hub-prompt-selector"); prSelect.innerHTML = ``; prompts.forEach(p => { const opt = document.createElement("option"); opt.value = p.id; opt.textContent = `${p.name} (v${p.version}) - ID: ${p.id}`; prSelect.appendChild(opt); }); // Populate historical runs selector const historySelect = document.getElementById("hub-history-run-selector"); historySelect.innerHTML = ``; runs.forEach(r => { const opt = document.createElement("option"); opt.value = r.id; opt.textContent = `Run ID: ${r.id} (Dataset ID: ${r.dataset_id}) - Status: ${r.status}`; historySelect.appendChild(opt); }); document.getElementById("hub-run-status-container").classList.add("hidden"); document.getElementById("hub-history-details-container").classList.add("hidden"); } catch (err) { console.error("Error loading hub config data:", err); } } let pollingInterval = null; async function handleTriggerEvaluation(e) { e.preventDefault(); const dsId = document.getElementById("hub-ds-selector").value; const prId = document.getElementById("hub-prompt-selector").value; // Collect multiselect models checkboxes const checkedBoxes = document.querySelectorAll("input[name='hub-models']:checked"); const models = Array.from(checkedBoxes).map(cb => cb.value); if (models.length === 0) { alert("Please select at least one LLM model provider to evaluate."); return; } const payload = { dataset_id: parseInt(dsId), prompt_id: prId ? parseInt(prId) : null, models: models }; try { const run = await fetchApi('/evaluations/trigger', { method: 'POST', body: JSON.stringify(payload) }); // Show status tracker UI const container = document.getElementById("hub-run-status-container"); container.classList.remove("hidden"); const statusEl = document.getElementById("hub-run-status"); const fillEl = document.getElementById("hub-run-progress"); const msgEl = document.getElementById("hub-run-message"); statusEl.textContent = run.status; fillEl.style.width = "10%"; msgEl.textContent = "Pipeline execution started in background. Polling server metrics..."; // Clear any old polls if (pollingInterval) clearInterval(pollingInterval); let progressPercent = 10; pollingInterval = setInterval(async () => { try { const polledRun = await fetchApi(`/evaluations/${run.id}`); statusEl.textContent = polledRun.status; if (polledRun.status === 'COMPLETED') { clearInterval(pollingInterval); fillEl.style.width = "100%"; msgEl.innerHTML = `Pipeline evaluation complete! Click 'Historical Execution Logs' to inspect outputs.`; } else if (polledRun.status === 'FAILED') { clearInterval(pollingInterval); fillEl.style.width = "100%"; fillEl.style.backgroundColor = "var(--accent-red)"; msgEl.innerHTML = `Run failed: ${polledRun.metrics?.error || 'Unknown executor error.'}`; } else { progressPercent = Math.min(progressPercent + 10, 95); fillEl.style.width = `${progressPercent}%`; } } catch (err) { console.error("Poller status checking failed:", err); } }, 1500); } catch (err) { alert(`Failed to trigger evaluation pipeline: ${err.message}`); } } async function loadHistoryRunDetails() { const runId = document.getElementById("hub-history-run-selector").value; if (!runId) { document.getElementById("hub-history-details-container").classList.add("hidden"); return; } try { const run = await fetchApi(`/evaluations/${runId}`); document.getElementById("hub-history-details-container").classList.remove("hidden"); document.getElementById("inspect-run-heading").textContent = `Run Metadata — Status: ${run.status}`; const dt = new Date(run.created_at); document.getElementById("inspect-run-timestamp").textContent = `Timestamp: ${dt.toLocaleString()} UTC`; const metrics = run.metrics || {}; document.getElementById("inspect-kpi-accuracy").textContent = metrics.average_accuracy ? metrics.average_accuracy.toFixed(2) : 'N/A'; document.getElementById("inspect-kpi-hallucination").textContent = metrics.hallucination_rate !== undefined ? `${(metrics.hallucination_rate * 100).toFixed(2)}%` : 'N/A'; document.getElementById("inspect-kpi-latency").textContent = metrics.average_latency_ms ? `${metrics.average_latency_ms.toFixed(0)} ms` : 'N/A'; document.getElementById("inspect-kpi-cost").textContent = metrics.total_cost ? `$${metrics.total_cost.toFixed(4)}` : 'N/A'; // Load granular results table const tbody = document.getElementById("inspect-run-results-table-body"); tbody.innerHTML = ""; const results = run.results || []; if (results.length === 0) { tbody.innerHTML = `No granular results generated for this run.`; return; } results.forEach(res => { const tr = document.createElement("tr"); const tc = res.test_case || {}; tr.innerHTML = ` ${res.test_case_id} ${res.model_name} ${escapeHtml(tc.question || 'N/A')} ${escapeHtml(tc.ground_truth || 'N/A')} ${escapeHtml(res.raw_output)} Acc: ${res.accuracy || 0}
Comp: ${res.completeness || 0}
Hall: ${res.hallucination || 0} ${escapeHtml(res.reason || '')} ${res.latency_ms?.toFixed(0) || 0} ms $${res.cost?.toFixed(5) || 0} `; tbody.appendChild(tr); }); } catch (err) { alert("Failed to load historical execution metrics."); } } // --- 5. RCA Agent & Security Auditor Operations --- async function loadRcaConfiguration() { try { const [runs, datasets] = await Promise.all([ fetchApi('/evaluations/'), fetchApi('/datasets/') ]); const completedRuns = runs.filter(r => r.status === 'COMPLETED'); // Populate RCA select const runSelect = document.getElementById("rca-run-selector"); runSelect.innerHTML = ``; completedRuns.forEach(r => { const opt = document.createElement("option"); opt.value = r.id; opt.textContent = `Run ID: ${r.id} (Dataset ID: ${r.dataset_id}) - Avg Score: ${r.metrics?.average_accuracy?.toFixed(2) || '0.00'}`; runSelect.appendChild(opt); }); // Populate Redteam Seed select const seedSelect = document.getElementById("rca-seed-selector"); seedSelect.innerHTML = ``; // Avoid showing already adversarial datasets const seedDatasets = datasets.filter(ds => !ds.name.includes("RedTeam-Adversarial")); seedDatasets.forEach(ds => { const opt = document.createElement("option"); opt.value = ds.id; opt.textContent = `${ds.name} (v${ds.version}) - ID: ${ds.id}`; seedSelect.appendChild(opt); }); document.getElementById("rca-report-result").classList.add("hidden"); document.getElementById("redteam-results-container").classList.add("hidden"); } catch (err) { console.error("Failed to load RCA/Red-team configuration selectors:", err); } } async function triggerRcaReport() { const runId = document.getElementById("rca-run-selector").value; if (!runId) { alert("Please select a completed run to analyze."); return; } const resultBox = document.getElementById("rca-report-result"); const outputMarkdown = document.getElementById("rca-report-markdown"); resultBox.classList.remove("hidden"); outputMarkdown.textContent = "Agent compilation executing failures in background... Generating report..."; try { const res = await fetchApi('/agents/rca', { method: 'POST', body: JSON.stringify({ run_id: parseInt(runId) }) }); // Render simple markdown response outputMarkdown.textContent = res.report || "No response received from agent."; } catch (err) { outputMarkdown.innerHTML = `Agent RCA pipeline execution failed. Details: ${err.message}`; } } async function triggerRedTeamSuite() { const dsId = document.getElementById("rca-seed-selector").value; if (!dsId) { alert("Please select a seed dataset to audit."); return; } const container = document.getElementById("redteam-results-container"); const labelEl = document.getElementById("redteam-dataset-details"); const tbody = document.getElementById("redteam-table-body"); container.classList.remove("hidden"); labelEl.textContent = "AI Red Team auditor synthesizing prompt variations..."; tbody.innerHTML = `Auditing seed items... Creating attacks...`; try { const res = await fetchApi('/agents/redteam', { method: 'POST', body: JSON.stringify({ dataset_id: parseInt(dsId) }) }); labelEl.textContent = `${res.adversarial_dataset_name} (ID: ${res.adversarial_dataset_id})`; tbody.innerHTML = ""; const cases = res.cases || []; if (cases.length === 0) { tbody.innerHTML = `No adversarial cases synthesized.`; return; } cases.forEach(c => { const tr = document.createElement("tr"); const attackType = c.meta_data?.type || 'N/A'; const originalQ = c.meta_data?.original_question || 'N/A'; tr.innerHTML = ` ${attackType} ${escapeHtml(c.question)} ${escapeHtml(c.ground_truth)} ${escapeHtml(originalQ)} `; tbody.appendChild(tr); }); } catch (err) { tbody.innerHTML = `Red Teaming generation failed: ${err.message}`; } } // --- 6. Cost Analytics Operations --- async function loadCostAnalytics() { try { const runs = await fetchApi('/evaluations/'); const completedRuns = runs.filter(r => r.status === 'COMPLETED'); // Accumulate models cost/value stats const modelMetrics = {}; completedRuns.forEach(run => { const results = run.results || []; results.forEach(res => { const m = res.model_name; if (!modelMetrics[m]) { modelMetrics[m] = { accuracy: [], cost: [], latency: [] }; } if (res.accuracy !== null) modelMetrics[m].accuracy.push(res.accuracy); modelMetrics[m].cost.push(res.cost || 0.0); modelMetrics[m].latency.push(res.latency_ms || 0.0); }); }); // Fallbacks if database is empty if (Object.keys(modelMetrics).length === 0) { Object.assign(modelMetrics, { "gpt-4o": { accuracy: [4.75], cost: [0.0125], latency: [1250.0] }, "claude-3-5-sonnet": { accuracy: [4.82], cost: [0.0095], latency: [1500.0] }, "gemini-1.5-pro": { accuracy: [4.60], cost: [0.0045], latency: [1900.0] }, "gemini-1.5-flash": { accuracy: [4.10], cost: [0.0003], latency: [650.0] }, "claude-3-haiku": { accuracy: [3.95], cost: [0.00025], latency: [450.0] } }); } const summaryRows = []; Object.keys(modelMetrics).forEach(model => { const accs = modelMetrics[model].accuracy; const costs = modelMetrics[model].cost; const lats = modelMetrics[model].latency; const avgAcc = accs.length ? accs.reduce((a,b)=>a+b,0)/accs.length : 0.0; const avgCost = costs.length ? costs.reduce((a,b)=>a+b,0)/costs.length : 0.0; const avgLat = lats.length ? lats.reduce((a,b)=>a+b,0)/lats.length : 0.0; const millidollarCost = avgCost * 1000; const efficiency = millidollarCost > 0 ? avgAcc / millidollarCost : 0.0; summaryRows.push({ model, accuracy: avgAcc, cost: avgCost, latency: avgLat, efficiency: efficiency }); }); // Render Bar Charts renderCostBarCharts(summaryRows); // Generate recommendations generateRoutingRecommendations(summaryRows); } catch (err) { console.error("Failed to load cost analytics metrics:", err); } } function renderCostBarCharts(summaryRows) { const models = summaryRows.map(r => r.model); const costs = summaryRows.map(r => r.cost); const efficiencies = summaryRows.map(r => r.efficiency); // Chart 1: Cost per Query (Log scale) const trace1 = { x: models, y: costs, type: 'bar', marker: { color: '#C27D38', line: { width: 1, color: '#2E221E' } } }; const layout1 = { yaxis: { title: 'Avg Cost ($/Query)', type: 'log', gridcolor: 'rgba(194, 125, 56, 0.1)', linecolor: 'rgba(194, 125, 56, 0.15)', tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, titlefont: { color: '#C27D38' } }, xaxis: { tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, linecolor: 'rgba(194, 125, 56, 0.15)' }, paper_bgcolor: 'rgba(0, 0, 0, 0)', plot_bgcolor: 'rgba(0, 0, 0, 0)', font: { color: '#2E221E', family: 'Plus Jakarta Sans, sans-serif' }, margin: { t: 20, b: 40, l: 60, r: 20 } }; Plotly.newPlot('cost-bar-chart', [trace1], layout1); // Chart 2: Efficiency ratio const trace2 = { x: models, y: efficiencies, type: 'bar', marker: { color: '#A64B2A', line: { width: 1, color: '#C27D38' } } }; const layout2 = { yaxis: { title: 'Accuracy per Millidollar', gridcolor: 'rgba(194, 125, 56, 0.1)', linecolor: 'rgba(194, 125, 56, 0.15)', tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, titlefont: { color: '#C27D38' } }, xaxis: { tickfont: { color: '#2E221E', family: 'JetBrains Mono, monospace' }, linecolor: 'rgba(194, 125, 56, 0.15)' }, paper_bgcolor: 'rgba(0, 0, 0, 0)', plot_bgcolor: 'rgba(0, 0, 0, 0)', font: { color: '#2E221E', family: 'Plus Jakarta Sans, sans-serif' }, margin: { t: 20, b: 40, l: 60, r: 20 } }; Plotly.newPlot('value-bar-chart', [trace2], layout2); } function generateRoutingRecommendations(summaryRows) { const listContainer = document.getElementById("cost-recommendations-list"); listContainer.innerHTML = ""; const recommendations = []; const sonnet = summaryRows.find(r => r.model === 'claude-3-5-sonnet'); const gpt4o = summaryRows.find(r => r.model === 'gpt-4o'); const flash = summaryRows.find(r => r.model === 'gemini-1.5-flash'); if (sonnet && gpt4o) { if (sonnet.accuracy >= gpt4o.accuracy * 0.95 && sonnet.cost < gpt4o.cost) { const pctCost = (sonnet.cost / gpt4o.cost) * 100; const pctQuality = (sonnet.accuracy / gpt4o.accuracy) * 100; recommendations.push( `Claude 3.5 Sonnet Optimization: Claude provides ${pctQuality.toFixed(0)}% of GPT-4o quality at ${pctCost.toFixed(0)}% of the cost. Action: Route general reasoning questions to Claude.` ); } } if (flash) { if (flash.accuracy >= 4.0) { recommendations.push( `Gemini 1.5 Flash Routing: Flash has a very high value-to-cost ratio ($${flash.cost.toFixed(6)} per query). Action: Run a pre-classifier routing simple classification and summarization queries directly to Gemini Flash, bypassing expensive frontier models.` ); } } if (recommendations.length === 0) { recommendations.push( `Model Selection Routing: Claude 3.5 Sonnet currently provides 98% of GPT-4o quality at 76% of the cost. Router recommendation is active.` ); } recommendations.forEach(rec => { const div = document.createElement("div"); div.className = "recommendation-card"; div.innerHTML = rec; listContainer.appendChild(div); }); } // --- Utilities --- function escapeHtml(str) { if (!str) return ''; return str.toString() .replace(/&/g, "&") .replace(//g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } // --- Product Guide Modal Handlers --- function openProductGuide() { document.getElementById("guide-modal").classList.remove("hidden"); } function closeProductGuide() { document.getElementById("guide-modal").classList.add("hidden"); } function closeProductGuideOnOverlay(e) { if (e.target === document.getElementById("guide-modal")) { closeProductGuide(); } } // --- Scroll Reveal Animations --- function initScrollReveal() { let revealQueue = []; let revealTimeout = null; function processQueue() { if (revealQueue.length === 0) { revealTimeout = null; return; } const el = revealQueue.shift(); el.classList.add("reveal-active"); revealTimeout = setTimeout(processQueue, 80); // Stagger animations using timers (80ms delay) } const observer = new IntersectionObserver((entries) => { entries.forEach(entry => { if (entry.isIntersecting) { const target = entry.target; if (!target.classList.contains("reveal-active") && !revealQueue.includes(target)) { revealQueue.push(target); observer.unobserve(target); } } }); if (revealQueue.length > 0 && !revealTimeout) { processQueue(); } }, { threshold: 0.05, rootMargin: "0px 0px -40px 0px" }); const targetElements = document.querySelectorAll(".card, .metric-card, .chart-container, .flex-row:not(.masthead), .kpi-card"); targetElements.forEach(el => observer.observe(el)); }