Release Notes — {formatVersion(currentRun.version)}
-
{VERSION_NOTES[currentRun.version].map((note, i) => (
- {note} ))}
import { useState, useEffect } from 'react' import MetricCard from './components/MetricCard' import EvolutionChart from './components/EvolutionChart' import LatencyStats from './components/LatencyStats' const MAJOR_NAMES = { 1: 'Violet', 2: 'Indigo', 3: 'Azure', 4: 'Amber', 5: 'Scarlet' } function formatVersion(version) { const match = version.match(/^v(\d+)\.(\d+)/) if (!match) return version const [, major, minor] = match const name = MAJOR_NAMES[Number(major)] ?? `v${major}` return `${name} (v${major}.${minor})` } const VERSION_NOTES = { 'v1.0.0': [ 'Baseline: hybrid BM25 (0.3) + ChromaDB dense (0.7) → RRF → TinyBERT rerank top-10→5', '50 eval pairs across multi-hop, comparative, negative, numeric, and edge-case question types', 'answer_correctness (LLM judge vs ground truth) adopted as primary metric — replaces circular faithfulness', 'Separate eval-dashboard deployed; per-message faithfulness badge removed from UI', ], 'v1.1.0': [ 'HyDE enabled in this eval run — but LIVE production config has HyDE off (free tier TPM constraint)', 'Live config: contextual retrieval ON at ingest (Euron model, no Groq impact), HyDE off, Multi-Query off', 'This run is the closest available proxy — actual live recall sits between v1.0 baseline (0.51) and this run (0.72)', 'Per-query Groq calls in production: 1–2 (condense_question + answer only) — safe under 6000 TPM free tier', ], 'v1.2.0': [ 'Multi-Query Retrieval: LLM generates 3 phrasings of each query at retrieval time', 'Retrieves for each phrasing, deduplicates by best rank, RRF fuses wider candidate pool', 'Reranker still scores against the original query', 'Result: +0.7% recall, relevancy dropped — Phase 1 (query-side) exhausted; root cause is chunk quality', ], 'v1.3.0': [ 'Best measured stack: HyDE + Multi-Query + Contextual Retrieval — recall 0.768, P@5 0.984', 'Contextual Retrieval at ingest time: LLM prepends 2-sentence context per chunk before embedding (+18% recall)', 'HyDE + Multi-Query add 2 extra Groq calls per query — total 4 calls/query at this config', 'NOT live: Groq free tier 6000 TPM causes 429 storms under this call volume. Needs paid tier or alternative provider.', 'Upgrade path: switch to a provider with higher free TPM (or paid Groq) → re-enable hyde_enabled + multi_query_enabled in config.yaml', ], } const METRIC_DEFS = [ { key: 'answer_correctness', label: 'Answer Correctness', description: 'Are key facts in the generated answer correct vs the reference?', methodology: 'LLM-as-Judge: llama-3.1-8b scores each answer 1–5 against a human-written ground truth. Prompt checks for key facts present and correct. Score normalized to 0–1 (÷5). Independent of retrieved docs — judge only sees answer + reference.', }, { key: 'answer_relevancy', label: 'Answer Relevancy', description: 'Does the answer actually address the question asked?', methodology: 'RAGAS metric. Generates N reverse questions from the answer using an LLM, embeds them, then measures cosine similarity to the original question embedding. High = answer stays on-topic. Low = vague, padded, or off-topic response. Does not require ground truth.', }, { key: 'context_recall', label: 'Context Recall', description: 'Did retrieval surface all the chunks needed to answer correctly?', methodology: 'RAGAS metric. Breaks the ground truth reference into individual sentences. For each sentence, an LLM checks whether it can be attributed to the retrieved context. Score = attributed sentences ÷ total ground truth sentences. Requires ground truth.', }, { key: 'precision_at_5', label: 'Precision@5', description: 'What fraction of the top-5 retrieved chunks were actually relevant?', methodology: 'Custom metric. For each of the 5 reranked chunks returned: checks if the source filename matches the expected document AND if relevant keywords from the eval pair appear in the chunk text. Score = matching chunks ÷ 5. No LLM call — deterministic.', }, ] function EmptyState() { return (
Run the eval script to generate the first benchmark:
python scripts/run_eval_versioned.py --version v2.0 --tag "baseline" --n 50
Retrieval & answer quality metrics
Release Notes — {formatVersion(currentRun.version)}