import { useState, useEffect } from 'react' import MetricCard from './components/MetricCard' import EvolutionChart from './components/EvolutionChart' import LatencyStats from './components/LatencyStats' const MAJOR_NAMES = { 1: 'Violet', 2: 'Indigo', 3: 'Azure', 4: 'Amber', 5: 'Scarlet' } function formatVersion(version) { const match = version.match(/^v(\d+)\.(\d+)/) if (!match) return version const [, major, minor] = match const name = MAJOR_NAMES[Number(major)] ?? `v${major}` return `${name} (v${major}.${minor})` } const VERSION_NOTES = { 'v1.0.0': [ 'Baseline: hybrid BM25 (0.3) + ChromaDB dense (0.7) → RRF → TinyBERT rerank top-10→5', '50 eval pairs across multi-hop, comparative, negative, numeric, and edge-case question types', 'answer_correctness (LLM judge vs ground truth) adopted as primary metric — replaces circular faithfulness', 'Separate eval-dashboard deployed; per-message faithfulness badge removed from UI', ], 'v1.1.0': [ 'HyDE enabled in this eval run — but LIVE production config has HyDE off (free tier TPM constraint)', 'Live config: contextual retrieval ON at ingest (Euron model, no Groq impact), HyDE off, Multi-Query off', 'This run is the closest available proxy — actual live recall sits between v1.0 baseline (0.51) and this run (0.72)', 'Per-query Groq calls in production: 1–2 (condense_question + answer only) — safe under 6000 TPM free tier', ], 'v1.2.0': [ 'Multi-Query Retrieval: LLM generates 3 phrasings of each query at retrieval time', 'Retrieves for each phrasing, deduplicates by best rank, RRF fuses wider candidate pool', 'Reranker still scores against the original query', 'Result: +0.7% recall, relevancy dropped — Phase 1 (query-side) exhausted; root cause is chunk quality', ], 'v1.3.0': [ 'Best measured stack: HyDE + Multi-Query + Contextual Retrieval — recall 0.768, P@5 0.984', 'Contextual Retrieval at ingest time: LLM prepends 2-sentence context per chunk before embedding (+18% recall)', 'HyDE + Multi-Query add 2 extra Groq calls per query — total 4 calls/query at this config', 'NOT live: Groq free tier 6000 TPM causes 429 storms under this call volume. Needs paid tier or alternative provider.', 'Upgrade path: switch to a provider with higher free TPM (or paid Groq) → re-enable hyde_enabled + multi_query_enabled in config.yaml', ], } const METRIC_DEFS = [ { key: 'answer_correctness', label: 'Answer Correctness', description: 'Are key facts in the generated answer correct vs the reference?', methodology: 'LLM-as-Judge: llama-3.1-8b scores each answer 1–5 against a human-written ground truth. Prompt checks for key facts present and correct. Score normalized to 0–1 (÷5). Independent of retrieved docs — judge only sees answer + reference.', }, { key: 'answer_relevancy', label: 'Answer Relevancy', description: 'Does the answer actually address the question asked?', methodology: 'RAGAS metric. Generates N reverse questions from the answer using an LLM, embeds them, then measures cosine similarity to the original question embedding. High = answer stays on-topic. Low = vague, padded, or off-topic response. Does not require ground truth.', }, { key: 'context_recall', label: 'Context Recall', description: 'Did retrieval surface all the chunks needed to answer correctly?', methodology: 'RAGAS metric. Breaks the ground truth reference into individual sentences. For each sentence, an LLM checks whether it can be attributed to the retrieved context. Score = attributed sentences ÷ total ground truth sentences. Requires ground truth.', }, { key: 'precision_at_5', label: 'Precision@5', description: 'What fraction of the top-5 retrieved chunks were actually relevant?', methodology: 'Custom metric. For each of the 5 reranked chunks returned: checks if the source filename matches the expected document AND if relevant keywords from the eval pair appear in the chunk text. Score = matching chunks ÷ 5. No LLM call — deterministic.', }, ] function EmptyState() { return (

No eval runs yet

Run the eval script to generate the first benchmark:

        python scripts/run_eval_versioned.py --version v2.0 --tag "baseline" --n 50
      
) } export default function App() { const [indexData, setIndexData] = useState([]) const [runs, setRuns] = useState([]) const [selectedVersion, setSelectedVersion] = useState(null) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) useEffect(() => { fetch('/data/index.json') .then(r => r.json()) .then(async (idx) => { if (!idx.length) { setLoading(false); return } const loaded = await Promise.all( idx.map(entry => fetch(`/data/runs/${entry.file}`).then(r => r.json())) ) setIndexData(idx) setRuns(loaded) setSelectedVersion(idx[idx.length - 1].version) setLoading(false) }) .catch(e => { setError(e.message); setLoading(false) }) }, []) const currentIdx = indexData.findIndex(e => e.version === selectedVersion) const currentRun = runs[currentIdx] ?? null const prevRun = currentIdx > 0 ? runs[currentIdx - 1] : null return (
{/* Header */}

Prism Eval Dashboard

Retrieval & answer quality metrics

{indexData.length > 0 && (
{indexData.length} run{indexData.length !== 1 ? 's' : ''}
)}
{loading && (
Loading...
)} {error && (
Failed to load eval data: {error}
)} {!loading && !error && !runs.length && } {currentRun && ( <> {/* Run meta */}
{formatVersion(currentRun.version)} {indexData[currentIdx]?.is_live && ( LIVE )} {currentRun.sample_count} samples · {new Date(currentRun.computed_at).toLocaleDateString('en-IN', { day: 'numeric', month: 'short', year: 'numeric' })} {currentRun.config && ( <> · HyDE: {currentRun.config.hyde_enabled ? 'on' : 'off'} · retrieve_k={currentRun.config.retrieve_k} )}
{/* Live proxy note */} {indexData[currentIdx]?.is_live && indexData[currentIdx]?.live_note && (
Production config note — {indexData[currentIdx].live_note}
)} {/* Blocked constraint warning */} {indexData[currentIdx]?.blocked_by && (
Not in production — {indexData[currentIdx].blocked_by}
)} {/* Release notes */} {VERSION_NOTES[currentRun.version] && (

Release Notes — {formatVersion(currentRun.version)}

    {VERSION_NOTES[currentRun.version].map((note, i) => (
  • {note}
  • ))}
)} {/* Metric cards */}
{METRIC_DEFS.map(m => ( ))}
{/* Evolution chart + latency */}
)}
) }