logatra-benchmark / index.html
CooledNik's picture
Publish benchmark report
05c474f verified
Raw History Blame Contribute Delete
77.2 kB
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Logatra Benchmark — Indic speech-to-text, measured</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&family=Plus+Jakarta+Sans:wght@300;400;500;600&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.9/dist/chart.umd.min.js"></script>
<script>(function(){var t=localStorage.getItem("nemukai-theme")||"light";document.documentElement.setAttribute("data-theme",t);})();</script>
<style>
:root{
--ink900:#141210;--ink800:#262220;--ink700:#3B352E;--ink600:#524A3E;
--ink500:#6D6455;--ink400:#8E8474;--ink300:#B0A695;--ink100:#DED5C3;
--ink50:#ECE4D4;--sand:#F2EBDD;
--kai500:#1F869A;--kai400:#3EA0B2;--kai300:#72BCC9;
--yuhi400:#E89A48;--momo400:#DB8E97;--momo500:#C96F79;
--laguna400:#42AD94;--hina400:#8670AA;--limu400:#619C4E;
--hairline:rgba(82,74,62,.38);
--bg:var(--ink900);--text-strong:var(--sand);--text:var(--ink50);--text-2:var(--ink100);
--text-muted:var(--ink300);--text-subtle:var(--ink400);--track:var(--ink700);--surface-2:var(--ink800);
--nav-bg:rgba(20,18,16,.92);--hover:rgba(59,53,46,.4);--head-bg:rgba(38,34,32,.7);
--mx-low:59,53,46;--mx-text-low:#DED5C3;--chart-text:#B0A695;--chart-grid:rgba(236,228,212,.07);
color-scheme:dark;
}
[data-theme="light"]{
--hairline:rgba(20,18,16,.14);
--bg:#F2EBDD;--text-strong:#141210;--text:#262220;--text-2:#3B352E;
--text-muted:#524A3E;--text-subtle:#6D6455;--track:#DED5C3;--surface-2:#ECE4D4;
--nav-bg:rgba(242,235,221,.9);--hover:rgba(222,213,195,.55);--head-bg:rgba(236,228,212,.75);
--mx-low:222,213,195;--mx-text-low:#141210;--chart-text:#524A3E;--chart-grid:rgba(20,18,16,.08);
color-scheme:light;
}
*{margin:0;padding:0;box-sizing:border-box}
html{scroll-behavior:smooth}
body{background:var(--bg);color:var(--text);transition:background-color .35s ease,color .35s ease;font-family:'Plus Jakarta Sans',sans-serif;
font-size:15px;line-height:1.6;position:relative;min-height:100vh}
body::before{content:'';position:fixed;inset:0;z-index:1;pointer-events:none;
background-image:url("data:image/svg+xml,<svg viewBox='0 0 256 256' xmlns='http://www.w3.org/2000/svg'><filter id='n'><feTurbulence type='fractalNoise' baseFrequency='0.75' numOctaves='4' stitchTiles='stitch'/><feColorMatrix type='saturate' values='0'/></filter><rect width='100%25' height='100%25' filter='url(%23n)'/></svg>");
background-size:256px;opacity:.11}
body::after{content:'';position:fixed;inset:0;z-index:1;pointer-events:none;
background-image:
url("data:image/svg+xml,<svg viewBox='0 0 220 220' xmlns='http://www.w3.org/2000/svg'><g stroke='%23F6F0E7' stroke-opacity='.03' stroke-width='.85' stroke-linecap='round' fill='none'><path d='M21 34l14 11'/><path d='M180 22l9 8'/><path d='M64 92l19 14'/><path d='M150 130l26 19'/><path d='M30 170l12 9'/><path d='M120 60l7 5'/><path d='M88 200l16 -3'/><path d='M200 90l-8 14'/><path d='M48 16l24 5'/><path d='M140 180l11 8'/><path d='M12 116l8 6'/><path d='M172 158l6 4'/></g></svg>"),
url("data:image/svg+xml,<svg viewBox='0 0 150 150' xmlns='http://www.w3.org/2000/svg'><g fill='%23FFFFFF' opacity='.04'><circle cx='14' cy='22' r='1.1'/><circle cx='66' cy='9' r='.8'/><circle cx='118' cy='40' r='1.3'/><circle cx='38' cy='80' r='.9'/><circle cx='96' cy='104' r='1.2'/><circle cx='140' cy='128' r='.8'/><circle cx='22' cy='132' r='1.4'/><circle cx='74' cy='58' r='.9'/></g></svg>");
background-size:220px,150px}
main,nav,footer{position:relative;z-index:2}
nav{display:flex;align-items:baseline;gap:24px;padding:18px 5vw;border-bottom:.5px solid var(--hairline);
position:sticky;top:0;background:var(--nav-bg);backdrop-filter:blur(6px);z-index:10}
nav .mark{font-family:'Instrument Serif',serif;font-size:21px;color:var(--text-strong)}
nav .mark span{color:var(--kai400)}
nav .links{margin-left:auto;display:flex;gap:22px;font-size:13px}
nav a{color:var(--text-muted);text-decoration:none;transition:color .15s ease-out}
nav a:hover{color:var(--text-strong)}
.wrap{max-width:1280px;margin:0 auto;padding:0 5vw}
header.hero{padding:84px 0 48px}
h1{font-family:'Instrument Serif',serif;font-weight:400;font-size:clamp(34px,5vw,58px);line-height:1.1;
letter-spacing:-.02em;color:var(--text-strong);max-width:780px}
h1 em{font-style:italic;color:var(--kai300)}
.hero p{margin-top:20px;max-width:640px;color:var(--text-muted);font-size:16px}
.statstrip{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));
border-top:.5px solid var(--hairline);border-bottom:.5px solid var(--hairline);margin-top:44px}
.statstrip div{padding:24px 20px;border-right:.5px solid var(--hairline)}
.statstrip div:last-child{border-right:none}
.statstrip b{display:block;font-family:'JetBrains Mono',monospace;font-weight:500;font-size:26px;color:var(--text-strong)}
.statstrip span{font-size:12.5px;color:var(--text-subtle)}
section{padding:76px 0 0}
h2{font-family:'Instrument Serif',serif;font-weight:400;font-size:30px;color:var(--text-strong);letter-spacing:-.01em}
.sub{color:var(--text-subtle);font-size:14px;margin:8px 0 26px;max-width:680px}
.reveal{opacity:0;transform:translateY(14px);transition:opacity .45s ease-out,transform .45s ease-out}
.reveal.in{opacity:1;transform:none}
@media (prefers-reduced-motion:reduce){.reveal{opacity:1;transform:none;transition:none}}
.toggle{display:inline-flex;border:.5px solid var(--hairline);border-radius:8px;overflow:hidden;font-size:13px;margin-bottom:18px}
.toggle button{background:none;border:none;color:var(--text-muted);padding:8px 18px;font-family:inherit;font-size:inherit;cursor:pointer;transition:background .15s,color .15s}
.toggle button.on{background:var(--track);color:var(--text-strong)}
.toggle button:focus-visible{outline:2px solid var(--kai400);outline-offset:-2px}
.hint{font-size:12.5px;color:var(--text-subtle);margin:-8px 0 18px}
/* leaderboard */
.lb{border:.5px solid var(--hairline)}
.lbrow{display:grid;grid-template-columns:44px 220px 1fr 110px 110px;align-items:center;
border-bottom:.5px solid var(--hairline);padding:0 8px;transition:background .15s}
.lbrow:last-child{border-bottom:none}
.lbrow:hover{background:var(--hover)}
.lbrow>div{padding:18px 12px}
.lbrow .rank{font-family:'Instrument Serif',serif;font-size:22px;color:var(--text-subtle);text-align:center}
.lbrow.first .rank{color:var(--yuhi400)}
.lbrow .who{font-weight:500;color:var(--text-strong);font-size:14.5px}
.lbrow .who small{display:block;font-weight:400;color:var(--text-subtle);font-size:11.5px}
.dot{display:inline-block;width:9px;height:9px;border-radius:50%;margin-right:9px;vertical-align:0}
.accbar{position:relative;height:30px;display:flex;align-items:center;gap:12px}
.accbar .bar{height:8px;border-radius:4px;flex:1;background:var(--track);overflow:hidden;position:relative}
.accbar .fill{position:absolute;inset:0 auto 0 0;width:0;border-radius:4px;transition:width .8s cubic-bezier(.25,.8,.35,1)}
.accbar b{font-family:'JetBrains Mono',monospace;font-weight:500;font-size:16px;color:var(--text-strong);min-width:62px;text-align:right}
.cell{font-family:'JetBrains Mono',monospace;font-size:12.5px;color:var(--text-muted);text-align:right}
.flag{font-size:10px;font-family:'Plus Jakarta Sans',sans-serif;color:var(--momo400);border:.5px solid rgba(201,111,121,.4);border-radius:4px;padding:1px 6px;margin-left:8px;vertical-align:1px}
.flag.note{color:var(--text-subtle);border-color:var(--hairline)}
/* matrix */
.matrix{border:.5px solid var(--hairline);overflow-x:auto}
.mx{display:grid;min-width:760px}
.mx>div{border-bottom:.5px solid var(--hairline);border-right:.5px solid var(--hairline);
padding:10px 12px;font-size:12.5px;display:flex;align-items:center;justify-content:center;text-align:center}
.mx .rowh{justify-content:flex-start;color:var(--text-2);font-weight:500;background:var(--head-bg)}
.mx .colh{color:var(--text-muted);background:var(--head-bg);font-weight:500}
.mx .v{font-family:'JetBrains Mono',monospace;font-weight:500;color:var(--ink900)}
/* charts */
.grid2{display:grid;grid-template-columns:1fr 1fr;border:.5px solid var(--hairline)}
.grid2>div{padding:32px;border-right:.5px solid var(--hairline);border-bottom:.5px solid var(--hairline);min-width:0}
.grid2>div:nth-child(2n){border-right:none}
.grid2>div:nth-last-child(-n+2){border-bottom:none}
.grid2 h3{font-size:15px;font-weight:600;color:var(--text-strong);margin-bottom:4px}
.grid2 .note{font-size:12.5px;color:var(--text-subtle);margin-bottom:18px}
.chartbox{position:relative;height:300px}
/* explorer */
.filters{display:flex;gap:10px;flex-wrap:wrap;margin-bottom:6px}
.filters select,.filters input{background:var(--surface-2);border:.5px solid var(--hairline);color:var(--text);
border-radius:8px;padding:8px 12px;font-family:inherit;font-size:13px}
.filters input{min-width:240px}
.filters select:focus,.filters input:focus{outline:2px solid var(--kai400);outline-offset:-1px}
.count{font-size:12.5px;color:var(--text-subtle);margin:10px 0 14px}
.kcase{border:.5px solid var(--hairline);border-bottom:none;cursor:pointer}
.kcase:last-of-type{border-bottom:.5px solid var(--hairline)}
.khead{display:flex;align-items:center;gap:14px;padding:14px 20px;transition:background .15s}
.kcase:hover .khead{background:var(--hover)}
.khead .kid{font-family:'JetBrains Mono',monospace;font-size:11.5px;color:var(--kai300);min-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.khead .tag{font-size:11px;color:var(--text-subtle);border:.5px solid var(--hairline);border-radius:4px;padding:1px 7px;white-space:nowrap}
.khead .tag.real{color:var(--laguna400);border-color:rgba(66,173,148,.4)}
.khead .chips{margin-left:auto;display:flex;gap:8px;flex-wrap:wrap}
.chip{font-family:'JetBrains Mono',monospace;font-size:11px;padding:2px 7px;border-radius:4px;color:var(--ink900);font-weight:500}
.kbody{display:none;border-top:.5px solid var(--hairline)}
.kcase.open .kbody{display:block}
.krow{display:grid;grid-template-columns:170px 1fr 80px;border-bottom:.5px solid var(--hairline)}
.krow:last-child{border-bottom:none}
.krow .who{padding:14px 0 14px 20px;font-size:12.5px;font-weight:500;color:var(--text-muted)}
.krow .txt{padding:14px 18px;font-size:14px;line-height:1.75;color:var(--text-2);border-left:.5px solid var(--hairline);word-break:break-word}
.krow .acc{padding:14px 16px 14px 0;text-align:right;font-family:'JetBrains Mono',monospace;font-size:12px;color:var(--text-muted)}
.krow.ref{background:rgba(31,134,154,.07)}
.krow.ref .who,.krow.ref .txt{color:var(--text-strong)}
.txt .sub-w{background:rgba(201,111,121,.28);border-radius:3px;padding:0 2px}
.txt .ins-w{background:rgba(232,154,72,.22);border-radius:3px;padding:0 2px}
.txt .del-w{color:var(--momo400);text-decoration:line-through;opacity:.55;padding:0 2px}
.txt .cutnote{display:block;margin-top:6px;font-size:11.5px;color:var(--momo400)}
.legend{display:flex;gap:18px;font-size:12px;color:var(--text-subtle);margin:14px 0}
.legend i{font-style:normal;padding:1px 6px;border-radius:3px}
.pager{display:flex;gap:10px;align-items:center;margin:20px 0 0;font-size:13px}
.pager button{background:var(--surface-2);border:.5px solid var(--hairline);color:var(--text-2);border-radius:8px;padding:7px 16px;cursor:pointer;font-family:inherit}
.pager button:disabled{opacity:.35;cursor:default}
.pager button:not(:disabled):hover{background:var(--track)}
.foot{margin-top:88px;border-top:.5px solid var(--hairline);padding:36px 5vw;font-size:12.5px;color:var(--text-subtle);display:flex;gap:32px;flex-wrap:wrap}
@media (max-width:900px){
.grid2{grid-template-columns:1fr}
.grid2>div{border-right:none}
.lbrow{grid-template-columns:34px 150px 1fr}
.lbrow .cell{display:none}
.krow{grid-template-columns:110px 1fr}
.krow .acc{grid-column:2;text-align:left;padding:0 18px 12px}
.khead .kid{min-width:120px}
}
[data-theme="light"] body::before{opacity:.07}
[data-theme="light"] body::after{background-image:url("data:image/svg+xml,<svg viewBox='0 0 130 130' xmlns='http://www.w3.org/2000/svg'><g fill='%23503918' opacity='.07'><circle cx='11' cy='18' r='1'/><circle cx='32' cy='44' r='1.3'/><circle cx='61' cy='22' r='.9'/><circle cx='78' cy='53' r='1.2'/><circle cx='108' cy='27' r='1.1'/><circle cx='17' cy='92' r='1.1'/><circle cx='43' cy='115' r='1.4'/><circle cx='70' cy='88' r='.8'/><circle cx='95' cy='113' r='1.2'/><circle cx='121' cy='96' r='1'/></g></svg>");background-size:130px}
nav{align-items:center}
nav .mark a{color:inherit;text-decoration:none}
nav .links{align-items:center}
.theme-toggle{display:inline-flex;align-items:center;gap:7px;padding:6px 12px 6px 9px;border-radius:20px;border:.5px solid var(--hairline);background:transparent;color:var(--text-muted);cursor:pointer;font-family:inherit;font-size:11px;font-weight:600;letter-spacing:.06em;text-transform:uppercase;transition:background .18s ease,border-color .18s ease,color .18s ease}
.theme-toggle:hover{background:var(--hover);border-color:var(--kai400);color:var(--text-strong)}
.theme-toggle:focus-visible{outline:2px solid var(--kai400);outline-offset:2px}
.theme-icon{width:14px;height:14px;fill:none;stroke:currentColor;stroke-width:1.5;stroke-linecap:round;stroke-linejoin:round}
@media (max-width:640px){nav .links a{display:none}.theme-label{display:none}}
</style>
</head>
<body>
<nav><div class="mark"><a href="/labs">nemukai</a> <span>/ logatra benchmark</span></div>
<div class="links"><a href="#leaderboard">leaderboard</a><a href="#matrix">languages</a><a href="#charts">charts</a><a href="#explorer">explorer</a><a href="#method">method</a>
<button type="button" class="theme-toggle" id="themeToggle" aria-label="Switch theme"><svg viewBox="0 0 24 24" class="theme-icon" id="themeIcon"></svg><span class="theme-label" id="themeLabel"></span></button></div></nav>
<main class="wrap">
<header class="hero">
<h1>How well do speech APIs actually hear <em>India</em>?</h1>
<p>Seven speech-to-text engines, tested on the same audio: read speech in 12 Indian languages, Hinglish conversations, noisy phone calls, and fresh real-world recordings no model has trained on. Every test case is inspectable below, ground truth against output, word by word.</p>
<div class="statstrip" id="statstrip"></div>
</header>
<section id="leaderboard" class="reveal">
<h2>Leaderboard</h2>
<p class="sub">Accuracy = words transcribed correctly, after normalizing away script choice (writing "start" vs "स्टार्ट" for the same spoken word is not an error). Computed on the exact same clips for every engine.</p>
<div class="toggle" role="tablist" id="splitToggle">
<button class="on" data-split="heldout">real-world 2026 audio</button>
<button data-split="public">standard datasets</button>
</div>
<p class="hint" id="splitHint"></p>
<div class="lb" id="board"></div>
</section>
<section id="matrix" class="reveal">
<h2>Accuracy by language</h2>
<p class="sub">Average per-clip accuracy for each engine on each test type. Darker teal = better. This is the map of who to route where.</p>
<div class="matrix"><div class="mx" id="mx"></div></div>
</section>
<section id="charts" class="reveal">
<h2>The shape of the gap</h2>
<div class="grid2">
<div><h3>Accuracy by test type</h3><div class="note">read speech vs Hinglish vs phone calls vs fresh real-world audio</div><div class="chartbox"><canvas id="cDataset"></canvas></div></div>
<div><h3>Accuracy vs speed</h3><div class="note">up and left is better · bubble size = cost per audio hour</div><div class="chartbox"><canvas id="cScatter"></canvas></div></div>
<div><h3>What an hour of audio costs</h3><div class="note">measured from per-call billing or list price</div><div class="chartbox"><canvas id="cCost"></canvas></div></div>
<div><h3>Processing speed</h3><div class="note">seconds of processing per second of audio (lower = faster)</div><div class="chartbox"><canvas id="cRtf"></canvas></div></div>
</div>
</section>
<section id="explorer" class="reveal">
<h2>Every test case, inspectable</h2>
<p class="note" style="margin:0 0 14px">Gramvaani phone-call cases are scored in the leaderboard but not shown here: that corpus is licensed for academic use only and does not permit redistributing its transcripts.</p>
<p class="sub">Ground truth on top, every engine's output below it. Differences are highlighted: <span class="sub-w" style="background:rgba(201,111,121,.28);border-radius:3px;padding:0 4px">wrong word</span> <span style="background:rgba(232,154,72,.22);border-radius:3px;padding:0 4px">added word</span> <span style="color:#DB8E97;text-decoration:line-through">missed word</span>. Click any case to expand.</p>
<div class="filters">
<select id="fDs"><option value="">all test types</option></select>
<select id="fLang"><option value="">all languages</option></select>
<select id="fSort">
<option value="disagree">most disagreement first</option>
<option value="hard">hardest first</option>
<option value="easy">easiest first</option>
<option value="id">by id</option>
</select>
<input id="fQ" type="search" placeholder="search transcripts…" aria-label="search transcripts">
</div>
<div class="count" id="fCount"></div>
<div id="caselist"></div>
<div class="pager">
<button id="pgPrev">← previous</button>
<span id="pgInfo"></span>
<button id="pgNext">next →</button>
</div>
</section>
<section id="method" class="reveal">
<h2>Method, honestly</h2>
<div class="grid2">
<div><h3>Two kinds of tests</h3><div class="note" style="margin:0"><b style="color:var(--sand)">Standard datasets</b> (FLEURS read speech, MUCS Hinglish, Gramvaani phone calls) are public; providers have almost certainly trained on them, so treat those scores as a flattering baseline. <b style="color:var(--sand)">Real-world 2026</b> is Creative-Commons YouTube audio published January–June 2026, after every model's training cutoff. Nobody could have memorized it. When the two disagree, trust the real-world number.</div></div>
<div><h3>Where ground truth comes from</h3><div class="note" style="margin:0">Standard datasets ship human transcripts. Real-world references come from multi-model consensus with a strict agreement gate; the 36 of 132 chunks that failed the automatic gate were adjudicated case by case (accept only on cross-model content agreement, every decision and rationale logged in the repo). Sarvam saarika, Gemini, and Whisper contributed consensus hypotheses, so their real-world scores are upper bounds (marked); the per-system exposure is quantified as self_ref_rate in the data. Saaras v3, gpt-audio, and Voxtral never saw the references: their scores are fully independent.</div></div>
<div><h3>The script problem</h3><div class="note" style="margin:0">Indian speech mixes languages mid-sentence, and engines legitimately disagree on script: "फॉन्ट साइज" and "font size" are the same spoken words. Headline accuracy romanizes both sides before comparing so script choice is not punished. The diff view still shows the raw outputs, so you can see script differences without being misled by them.</div></div>
<div><h3>Models as run</h3><div class="note" style="margin:0" id="modelnote"></div></div>
</div>
</section>
</main>
<footer class="foot"><div>logatra benchmark · nemukai · 2026</div><p>Raw per-call responses are cached and the whole report regenerates offline. Accuracy floors at 0% (a looping engine cannot score negative, it just fails).</p></footer>
<script src="cases.js"></script>
<script>
const DATA = {"by_provider": {"sarvam|public": {"samples": 7229, "wer": 0.3602, "cer": 0.2453, "wer_script_agnostic": 0.3438, "wer_median": 0.3333, "self_ref_rate": 0.0, "latency_median_s": 1.09, "latency_p90_s": 3.759, "rtf_median": 0.1546, "cost_usd_total": 9.1556, "cost_usd_per_hour": 0.539, "audio_minutes": 1019.2}, "sarvam-saaras|public": {"samples": 7199, "wer": 0.3345, "cer": 0.2316, "wer_script_agnostic": 0.3182, "wer_median": 0.3043, "self_ref_rate": 0.0, "latency_median_s": 1.062, "latency_p90_s": 3.724, "rtf_median": 0.153, "cost_usd_total": 9.1165, "cost_usd_per_hour": 0.539, "audio_minutes": 1014.9}, "sarvam-saaras-v4|public": {"samples": 7230, "wer": 0.3006, "cer": 0.2071, "wer_script_agnostic": 0.2864, "wer_median": 0.2727, "self_ref_rate": 0.0, "latency_median_s": 0.559, "latency_p90_s": 2.88, "rtf_median": 0.0854, "cost_usd_total": 9.1571, "cost_usd_per_hour": 0.539, "audio_minutes": 1019.4}, "whisper|public": {"samples": 7238, "wer": 0.6247, "cer": 0.4087, "wer_script_agnostic": 0.5984, "wer_median": 0.6, "self_ref_rate": 0.0, "latency_median_s": 2.381, "latency_p90_s": 5.919, "rtf_median": 0.3242, "cost_usd_total": 6.1695, "cost_usd_per_hour": 0.36, "audio_minutes": 1028.1}, "gemini|public": {"samples": 799, "wer": 0.2081, "cer": 0.0927, "wer_script_agnostic": 0.2026, "wer_median": 0.1765, "self_ref_rate": 0.0, "latency_median_s": 3.164, "latency_p90_s": 4.444, "rtf_median": 0.3024, "cost_usd_total": 1.1236, "cost_usd_per_hour": 0.4451, "audio_minutes": 151.5}, "voxtral|public": {"samples": 800, "wer": 1.7363, "cer": 1.4727, "wer_script_agnostic": 1.7295, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.304, "latency_p90_s": 2.031, "rtf_median": 0.1214, "cost_usd_total": 0.8919, "cost_usd_per_hour": 0.3532, "audio_minutes": 151.5}, "gpt-audio|public": {"samples": 336, "wer": 0.5415, "cer": 0.3983, "wer_script_agnostic": 0.5362, "wer_median": 0.4242, "self_ref_rate": 0.0, "latency_median_s": 1.768, "latency_p90_s": 2.72, "rtf_median": 0.1903, "cost_usd_total": 1.2741, "cost_usd_per_hour": 1.3927, "audio_minutes": 54.9}, "sarvam|heldout": {"samples": 132, "wer": 0.1972, "cer": 0.1847, "wer_script_agnostic": 0.1814, "wer_median": 0.0, "self_ref_rate": 0.5758, "latency_median_s": 1.671, "latency_p90_s": 3.758, "rtf_median": 0.0746, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras|heldout": {"samples": 132, "wer": 0.2275, "cer": 0.202, "wer_script_agnostic": 0.2101, "wer_median": 0.1061, "self_ref_rate": 0.0, "latency_median_s": 1.529, "latency_p90_s": 4.083, "rtf_median": 0.0695, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras-v4|heldout": {"samples": 132, "wer": 0.258, "cer": 0.2247, "wer_script_agnostic": 0.2379, "wer_median": 0.1343, "self_ref_rate": 0.0, "latency_median_s": 0.739, "latency_p90_s": 0.953, "rtf_median": 0.0336, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "whisper|heldout": {"samples": 132, "wer": 0.4761, "cer": 0.3546, "wer_script_agnostic": 0.4589, "wer_median": 0.3898, "self_ref_rate": 0.0455, "latency_median_s": 5.911, "latency_p90_s": 13.112, "rtf_median": 0.265, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.36, "audio_minutes": 48.5}, "gemini|heldout": {"samples": 132, "wer": 0.2217, "cer": 0.1993, "wer_script_agnostic": 0.2048, "wer_median": 0.1167, "self_ref_rate": 0.3788, "latency_median_s": 3.713, "latency_p90_s": 4.851, "rtf_median": 0.1673, "cost_usd_total": 0.3471, "cost_usd_per_hour": 0.4295, "audio_minutes": 48.5}, "voxtral|heldout": {"samples": 132, "wer": 0.7198, "cer": 0.6632, "wer_script_agnostic": 0.7024, "wer_median": 0.2656, "self_ref_rate": 0.0, "latency_median_s": 1.931, "latency_p90_s": 2.688, "rtf_median": 0.0852, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.3601, "audio_minutes": 48.5}, "gpt-audio|heldout": {"samples": 132, "wer": 0.2852, "cer": 0.2414, "wer_script_agnostic": 0.2703, "wer_median": 0.1977, "self_ref_rate": 0.0, "latency_median_s": 2.661, "latency_p90_s": 3.435, "rtf_median": 0.1188, "cost_usd_total": 1.0902, "cost_usd_per_hour": 1.3489, "audio_minutes": 48.5}}, "by_provider_comparable": {"sarvam|public": {"samples": 324, "wer": 0.3731, "cer": 0.21, "wer_script_agnostic": 0.3622, "wer_median": 0.3182, "self_ref_rate": 0.0, "latency_median_s": 1.019, "latency_p90_s": 3.617, "rtf_median": 0.1163, "cost_usd_total": 0.4702, "cost_usd_per_hour": 0.539, "audio_minutes": 52.3}, "sarvam-saaras|public": {"samples": 324, "wer": 0.3231, "cer": 0.2048, "wer_script_agnostic": 0.3119, "wer_median": 0.2857, "self_ref_rate": 0.0, "latency_median_s": 1.076, "latency_p90_s": 3.664, "rtf_median": 0.1221, "cost_usd_total": 0.4702, "cost_usd_per_hour": 0.539, "audio_minutes": 52.3}, "sarvam-saaras-v4|public": {"samples": 324, "wer": 0.2612, "cer": 0.1621, "wer_script_agnostic": 0.2526, "wer_median": 0.2174, "self_ref_rate": 0.0, "latency_median_s": 0.588, "latency_p90_s": 2.903, "rtf_median": 0.078, "cost_usd_total": 0.4702, "cost_usd_per_hour": 0.539, "audio_minutes": 52.3}, "whisper|public": {"samples": 324, "wer": 0.7834, "cer": 0.5552, "wer_script_agnostic": 0.767, "wer_median": 0.7931, "self_ref_rate": 0.0, "latency_median_s": 3.276, "latency_p90_s": 10.133, "rtf_median": 0.3782, "cost_usd_total": 0.3141, "cost_usd_per_hour": 0.36, "audio_minutes": 52.3}, "gemini|public": {"samples": 324, "wer": 0.2646, "cer": 0.1602, "wer_script_agnostic": 0.255, "wer_median": 0.2333, "self_ref_rate": 0.0, "latency_median_s": 2.964, "latency_p90_s": 4.132, "rtf_median": 0.3408, "cost_usd_total": 0.3819, "cost_usd_per_hour": 0.4378, "audio_minutes": 52.3}, "voxtral|public": {"samples": 324, "wer": 2.9376, "cer": 2.6817, "wer_script_agnostic": 2.9293, "wer_median": 0.8333, "self_ref_rate": 0.0, "latency_median_s": 1.296, "latency_p90_s": 2.242, "rtf_median": 0.1411, "cost_usd_total": 0.3127, "cost_usd_per_hour": 0.3584, "audio_minutes": 52.3}, "gpt-audio|public": {"samples": 324, "wer": 0.5353, "cer": 0.3939, "wer_script_agnostic": 0.5298, "wer_median": 0.4167, "self_ref_rate": 0.0, "latency_median_s": 1.756, "latency_p90_s": 2.72, "rtf_median": 0.1903, "cost_usd_total": 1.2159, "cost_usd_per_hour": 1.3937, "audio_minutes": 52.3}, "sarvam|heldout": {"samples": 132, "wer": 0.1972, "cer": 0.1847, "wer_script_agnostic": 0.1814, "wer_median": 0.0, "self_ref_rate": 0.5758, "latency_median_s": 1.671, "latency_p90_s": 3.758, "rtf_median": 0.0746, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras|heldout": {"samples": 132, "wer": 0.2275, "cer": 0.202, "wer_script_agnostic": 0.2101, "wer_median": 0.1061, "self_ref_rate": 0.0, "latency_median_s": 1.529, "latency_p90_s": 4.083, "rtf_median": 0.0695, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras-v4|heldout": {"samples": 132, "wer": 0.258, "cer": 0.2247, "wer_script_agnostic": 0.2379, "wer_median": 0.1343, "self_ref_rate": 0.0, "latency_median_s": 0.739, "latency_p90_s": 0.953, "rtf_median": 0.0336, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "whisper|heldout": {"samples": 132, "wer": 0.4761, "cer": 0.3546, "wer_script_agnostic": 0.4589, "wer_median": 0.3898, "self_ref_rate": 0.0455, "latency_median_s": 5.911, "latency_p90_s": 13.112, "rtf_median": 0.265, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.36, "audio_minutes": 48.5}, "gemini|heldout": {"samples": 132, "wer": 0.2217, "cer": 0.1993, "wer_script_agnostic": 0.2048, "wer_median": 0.1167, "self_ref_rate": 0.3788, "latency_median_s": 3.713, "latency_p90_s": 4.851, "rtf_median": 0.1673, "cost_usd_total": 0.3471, "cost_usd_per_hour": 0.4295, "audio_minutes": 48.5}, "voxtral|heldout": {"samples": 132, "wer": 0.7198, "cer": 0.6632, "wer_script_agnostic": 0.7024, "wer_median": 0.2656, "self_ref_rate": 0.0, "latency_median_s": 1.931, "latency_p90_s": 2.688, "rtf_median": 0.0852, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.3601, "audio_minutes": 48.5}, "gpt-audio|heldout": {"samples": 132, "wer": 0.2852, "cer": 0.2414, "wer_script_agnostic": 0.2703, "wer_median": 0.1977, "self_ref_rate": 0.0, "latency_median_s": 2.661, "latency_p90_s": 3.435, "rtf_median": 0.1188, "cost_usd_total": 1.0902, "cost_usd_per_hour": 1.3489, "audio_minutes": 48.5}}, "comparable_counts": {"public": 324, "heldout": 132}, "by_provider_dataset": {"sarvam|fleurs": {"samples": 1189, "wer": 0.3325, "cer": 0.1723, "wer_script_agnostic": 0.3304, "wer_median": 0.1923, "self_ref_rate": 0.0, "latency_median_s": 1.102, "latency_p90_s": 3.778, "rtf_median": 0.0985, "cost_usd_total": 2.1888, "cost_usd_per_hour": 0.539, "audio_minutes": 243.7}, "sarvam-saaras|fleurs": {"samples": 1179, "wer": 0.2879, "cer": 0.1603, "wer_script_agnostic": 0.2859, "wer_median": 0.1765, "self_ref_rate": 0.0, "latency_median_s": 1.14, "latency_p90_s": 3.744, "rtf_median": 0.1029, "cost_usd_total": 2.1696, "cost_usd_per_hour": 0.539, "audio_minutes": 241.5}, "sarvam-saaras-v4|fleurs": {"samples": 1189, "wer": 0.1903, "cer": 0.0849, "wer_script_agnostic": 0.1883, "wer_median": 0.1282, "self_ref_rate": 0.0, "latency_median_s": 0.601, "latency_p90_s": 2.935, "rtf_median": 0.0569, "cost_usd_total": 2.1888, "cost_usd_per_hour": 0.539, "audio_minutes": 243.7}, "whisper|fleurs": {"samples": 1200, "wer": 0.8384, "cer": 0.5546, "wer_script_agnostic": 0.8269, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 4.307, "latency_p90_s": 12.512, "rtf_median": 0.386, "cost_usd_total": 1.5014, "cost_usd_per_hour": 0.36, "audio_minutes": 250.2}, "gemini|fleurs": {"samples": 600, "wer": 0.1709, "cer": 0.0474, "wer_script_agnostic": 0.1686, "wer_median": 0.1429, "self_ref_rate": 0.0, "latency_median_s": 3.222, "latency_p90_s": 4.486, "rtf_median": 0.28, "cost_usd_total": 0.9283, "cost_usd_per_hour": 0.445, "audio_minutes": 125.2}, "voxtral|fleurs": {"samples": 600, "wer": 0.9494, "cer": 0.6914, "wer_script_agnostic": 0.9447, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.326, "latency_p90_s": 1.973, "rtf_median": 0.1128, "cost_usd_total": 0.7308, "cost_usd_per_hour": 0.3503, "audio_minutes": 125.2}, "gpt-audio|fleurs": {"samples": 136, "wer": 0.3915, "cer": 0.2059, "wer_script_agnostic": 0.3915, "wer_median": 0.3333, "self_ref_rate": 0.0, "latency_median_s": 1.965, "latency_p90_s": 2.837, "rtf_median": 0.1634, "cost_usd_total": 0.6524, "cost_usd_per_hour": 1.3719, "audio_minutes": 28.5}, "sarvam|mucs_hinglish": {"samples": 3126, "wer": 0.5476, "cer": 0.5014, "wer_script_agnostic": 0.5282, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 1.014, "latency_p90_s": 3.655, "rtf_median": 0.2099, "cost_usd_total": 2.7663, "cost_usd_per_hour": 0.539, "audio_minutes": 307.9}, "sarvam-saaras|mucs_hinglish": {"samples": 3111, "wer": 0.5416, "cer": 0.5041, "wer_script_agnostic": 0.5217, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 1.02, "latency_p90_s": 3.665, "rtf_median": 0.2117, "cost_usd_total": 2.7543, "cost_usd_per_hour": 0.539, "audio_minutes": 306.6}, "sarvam-saaras-v4|mucs_hinglish": {"samples": 3127, "wer": 0.5061, "cer": 0.4699, "wer_script_agnostic": 0.4899, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 0.526, "latency_p90_s": 2.83, "rtf_median": 0.1132, "cost_usd_total": 2.7678, "cost_usd_per_hour": 0.539, "audio_minutes": 308.1}, "whisper|mucs_hinglish": {"samples": 3129, "wer": 0.5746, "cer": 0.4765, "wer_script_agnostic": 0.5514, "wer_median": 0.5417, "self_ref_rate": 0.0, "latency_median_s": 1.942, "latency_p90_s": 3.56, "rtf_median": 0.3635, "cost_usd_total": 1.8626, "cost_usd_per_hour": 0.3601, "audio_minutes": 310.4}, "gemini|mucs_hinglish": {"samples": 99, "wer": 0.4906, "cer": 0.4781, "wer_script_agnostic": 0.4795, "wer_median": 0.5294, "self_ref_rate": 0.0, "latency_median_s": 2.83, "latency_p90_s": 3.887, "rtf_median": 0.587, "cost_usd_total": 0.076, "cost_usd_per_hour": 0.4558, "audio_minutes": 10.0}, "voxtral|mucs_hinglish": {"samples": 100, "wer": 9.2611, "cer": 9.6549, "wer_script_agnostic": 9.2543, "wer_median": 1.125, "self_ref_rate": 0.0, "latency_median_s": 1.237, "latency_p90_s": 3.443, "rtf_median": 0.2287, "cost_usd_total": 0.0663, "cost_usd_per_hour": 0.3949, "audio_minutes": 10.1}, "gpt-audio|mucs_hinglish": {"samples": 100, "wer": 1.0323, "cer": 0.9865, "wer_script_agnostic": 1.0298, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.59, "latency_p90_s": 3.037, "rtf_median": 0.3236, "cost_usd_total": 0.244, "cost_usd_per_hour": 1.4531, "audio_minutes": 10.1}, "sarvam|gramvaani": {"samples": 2914, "wer": 0.2754, "cer": 0.1405, "wer_script_agnostic": 0.2559, "wer_median": 0.2581, "self_ref_rate": 0.0, "latency_median_s": 1.132, "latency_p90_s": 3.852, "rtf_median": 0.1231, "cost_usd_total": 4.2005, "cost_usd_per_hour": 0.5389, "audio_minutes": 467.7}, "sarvam-saaras|gramvaani": {"samples": 2909, "wer": 0.2461, "cer": 0.1184, "wer_script_agnostic": 0.227, "wer_median": 0.2308, "self_ref_rate": 0.0, "latency_median_s": 1.053, "latency_p90_s": 3.754, "rtf_median": 0.1136, "cost_usd_total": 4.1926, "cost_usd_per_hour": 0.5389, "audio_minutes": 466.8}, "sarvam-saaras-v4|gramvaani": {"samples": 2914, "wer": 0.233, "cer": 0.1146, "wer_script_agnostic": 0.2159, "wer_median": 0.2188, "self_ref_rate": 0.0, "latency_median_s": 0.572, "latency_p90_s": 2.894, "rtf_median": 0.0649, "cost_usd_total": 4.2005, "cost_usd_per_hour": 0.5389, "audio_minutes": 467.7}, "whisper|gramvaani": {"samples": 2909, "wer": 0.5808, "cer": 0.3274, "wer_script_agnostic": 0.5482, "wer_median": 0.5862, "self_ref_rate": 0.0, "latency_median_s": 2.607, "latency_p90_s": 5.38, "rtf_median": 0.2767, "cost_usd_total": 2.8055, "cost_usd_per_hour": 0.36, "audio_minutes": 467.6}, "gemini|gramvaani": {"samples": 100, "wer": 0.2529, "cer": 0.1291, "wer_script_agnostic": 0.235, "wer_median": 0.24, "self_ref_rate": 0.0, "latency_median_s": 3.192, "latency_p90_s": 4.506, "rtf_median": 0.3373, "cost_usd_total": 0.1193, "cost_usd_per_hour": 0.4396, "audio_minutes": 16.3}, "voxtral|gramvaani": {"samples": 100, "wer": 1.9666, "cer": 1.3767, "wer_script_agnostic": 1.9499, "wer_median": 0.4167, "self_ref_rate": 0.0, "latency_median_s": 1.285, "latency_p90_s": 2.146, "rtf_median": 0.1345, "cost_usd_total": 0.0947, "cost_usd_per_hour": 0.3491, "audio_minutes": 16.3}, "gpt-audio|gramvaani": {"samples": 100, "wer": 0.4701, "cer": 0.3257, "wer_script_agnostic": 0.458, "wer_median": 0.4, "self_ref_rate": 0.0, "latency_median_s": 1.71, "latency_p90_s": 2.451, "rtf_median": 0.174, "cost_usd_total": 0.3777, "cost_usd_per_hour": 1.392, "audio_minutes": 16.3}, "sarvam|heldout_yt": {"samples": 132, "wer": 0.1972, "cer": 0.1847, "wer_script_agnostic": 0.1814, "wer_median": 0.0, "self_ref_rate": 0.5758, "latency_median_s": 1.671, "latency_p90_s": 3.758, "rtf_median": 0.0746, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras|heldout_yt": {"samples": 132, "wer": 0.2275, "cer": 0.202, "wer_script_agnostic": 0.2101, "wer_median": 0.1061, "self_ref_rate": 0.0, "latency_median_s": 1.529, "latency_p90_s": 4.083, "rtf_median": 0.0695, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "sarvam-saaras-v4|heldout_yt": {"samples": 132, "wer": 0.258, "cer": 0.2247, "wer_script_agnostic": 0.2379, "wer_median": 0.1343, "self_ref_rate": 0.0, "latency_median_s": 0.739, "latency_p90_s": 0.953, "rtf_median": 0.0336, "cost_usd_total": 0.4356, "cost_usd_per_hour": 0.5389, "audio_minutes": 48.5}, "whisper|heldout_yt": {"samples": 132, "wer": 0.4761, "cer": 0.3546, "wer_script_agnostic": 0.4589, "wer_median": 0.3898, "self_ref_rate": 0.0455, "latency_median_s": 5.911, "latency_p90_s": 13.112, "rtf_median": 0.265, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.36, "audio_minutes": 48.5}, "gemini|heldout_yt": {"samples": 132, "wer": 0.2217, "cer": 0.1993, "wer_script_agnostic": 0.2048, "wer_median": 0.1167, "self_ref_rate": 0.3788, "latency_median_s": 3.713, "latency_p90_s": 4.851, "rtf_median": 0.1673, "cost_usd_total": 0.3471, "cost_usd_per_hour": 0.4295, "audio_minutes": 48.5}, "voxtral|heldout_yt": {"samples": 132, "wer": 0.7198, "cer": 0.6632, "wer_script_agnostic": 0.7024, "wer_median": 0.2656, "self_ref_rate": 0.0, "latency_median_s": 1.931, "latency_p90_s": 2.688, "rtf_median": 0.0852, "cost_usd_total": 0.291, "cost_usd_per_hour": 0.3601, "audio_minutes": 48.5}, "gpt-audio|heldout_yt": {"samples": 132, "wer": 0.2852, "cer": 0.2414, "wer_script_agnostic": 0.2703, "wer_median": 0.1977, "self_ref_rate": 0.0, "latency_median_s": 2.661, "latency_p90_s": 3.435, "rtf_median": 0.1188, "cost_usd_total": 1.0902, "cost_usd_per_hour": 1.3489, "audio_minutes": 48.5}}, "by_provider_language": {"sarvam|as": {"samples": 99, "wer": 0.9168, "cer": 0.3671, "wer_script_agnostic": 0.9168, "wer_median": 0.8947, "self_ref_rate": 0.0, "latency_median_s": 1.049, "latency_p90_s": 1.903, "rtf_median": 0.0941, "cost_usd_total": 0.1766, "cost_usd_per_hour": 0.539, "audio_minutes": 19.7}, "sarvam-saaras|as": {"samples": 92, "wer": 0.5346, "cer": 0.3189, "wer_script_agnostic": 0.5346, "wer_median": 0.4074, "self_ref_rate": 0.0, "latency_median_s": 1.132, "latency_p90_s": 1.58, "rtf_median": 0.0997, "cost_usd_total": 0.1614, "cost_usd_per_hour": 0.539, "audio_minutes": 18.0}, "sarvam-saaras-v4|as": {"samples": 99, "wer": 0.2537, "cer": 0.0923, "wer_script_agnostic": 0.2537, "wer_median": 0.2, "self_ref_rate": 0.0, "latency_median_s": 0.617, "latency_p90_s": 0.873, "rtf_median": 0.0555, "cost_usd_total": 0.1766, "cost_usd_per_hour": 0.539, "audio_minutes": 19.7}, "whisper|as": {"samples": 100, "wer": 1.1083, "cer": 0.7468, "wer_script_agnostic": 1.1083, "wer_median": 1.0556, "self_ref_rate": 0.0, "latency_median_s": 5.344, "latency_p90_s": 15.67, "rtf_median": 0.5338, "cost_usd_total": 0.1212, "cost_usd_per_hour": 0.36, "audio_minutes": 20.2}, "gemini|as": {"samples": 50, "wer": 0.2004, "cer": 0.0571, "wer_script_agnostic": 0.2004, "wer_median": 0.2, "self_ref_rate": 0.0, "latency_median_s": 3.029, "latency_p90_s": 4.177, "rtf_median": 0.2706, "cost_usd_total": 0.0793, "cost_usd_per_hour": 0.4492, "audio_minutes": 10.6}, "voxtral|as": {"samples": 50, "wer": 1.2326, "cer": 0.9643, "wer_script_agnostic": 1.2326, "wer_median": 1.25, "self_ref_rate": 0.0, "latency_median_s": 1.339, "latency_p90_s": 2.216, "rtf_median": 0.1197, "cost_usd_total": 0.0623, "cost_usd_per_hour": 0.3528, "audio_minutes": 10.6}, "gpt-audio|as": {"samples": 50, "wer": 0.5526, "cer": 0.3211, "wer_script_agnostic": 0.5526, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 2.154, "latency_p90_s": 2.995, "rtf_median": 0.1643, "cost_usd_total": 0.2429, "cost_usd_per_hour": 1.3757, "audio_minutes": 10.6}, "sarvam|bn": {"samples": 100, "wer": 0.15, "cer": 0.0375, "wer_script_agnostic": 0.15, "wer_median": 0.1304, "self_ref_rate": 0.0, "latency_median_s": 0.929, "latency_p90_s": 1.183, "rtf_median": 0.0751, "cost_usd_total": 0.2002, "cost_usd_per_hour": 0.539, "audio_minutes": 22.3}, "sarvam-saaras|bn": {"samples": 100, "wer": 0.1285, "cer": 0.0269, "wer_script_agnostic": 0.1285, "wer_median": 0.1176, "self_ref_rate": 0.0, "latency_median_s": 1.053, "latency_p90_s": 3.914, "rtf_median": 0.0867, "cost_usd_total": 0.2002, "cost_usd_per_hour": 0.539, "audio_minutes": 22.3}, "sarvam-saaras-v4|bn": {"samples": 100, "wer": 0.1145, "cer": 0.0308, "wer_script_agnostic": 0.1145, "wer_median": 0.0909, "self_ref_rate": 0.0, "latency_median_s": 1.054, "latency_p90_s": 3.005, "rtf_median": 0.0888, "cost_usd_total": 0.2002, "cost_usd_per_hour": 0.539, "audio_minutes": 22.3}, "whisper|bn": {"samples": 100, "wer": 1.1344, "cer": 0.8831, "wer_script_agnostic": 1.1344, "wer_median": 1.0556, "self_ref_rate": 0.0, "latency_median_s": 5.697, "latency_p90_s": 14.718, "rtf_median": 0.4739, "cost_usd_total": 0.1337, "cost_usd_per_hour": 0.36, "audio_minutes": 22.3}, "gemini|bn": {"samples": 50, "wer": 0.1421, "cer": 0.0311, "wer_script_agnostic": 0.1421, "wer_median": 0.12, "self_ref_rate": 0.0, "latency_median_s": 2.936, "latency_p90_s": 4.06, "rtf_median": 0.2166, "cost_usd_total": 0.0737, "cost_usd_per_hour": 0.3881, "audio_minutes": 11.4}, "voxtral|bn": {"samples": 50, "wer": 0.5301, "cer": 0.331, "wer_script_agnostic": 0.5301, "wer_median": 0.4, "self_ref_rate": 0.0, "latency_median_s": 1.443, "latency_p90_s": 2.218, "rtf_median": 0.1035, "cost_usd_total": 0.0665, "cost_usd_per_hour": 0.3503, "audio_minutes": 11.4}, "gpt-audio|bn": {"samples": 50, "wer": 0.2583, "cer": 0.1067, "wer_script_agnostic": 0.2583, "wer_median": 0.2308, "self_ref_rate": 0.0, "latency_median_s": 2.004, "latency_p90_s": 2.956, "rtf_median": 0.1508, "cost_usd_total": 0.2565, "cost_usd_per_hour": 1.3507, "audio_minutes": 11.4}, "sarvam|gu": {"samples": 100, "wer": 0.1876, "cer": 0.0509, "wer_script_agnostic": 0.1876, "wer_median": 0.1538, "self_ref_rate": 0.0, "latency_median_s": 1.132, "latency_p90_s": 3.771, "rtf_median": 0.122, "cost_usd_total": 0.1594, "cost_usd_per_hour": 0.539, "audio_minutes": 17.7}, "sarvam-saaras|gu": {"samples": 99, "wer": 0.1906, "cer": 0.0536, "wer_script_agnostic": 0.1906, "wer_median": 0.1579, "self_ref_rate": 0.0, "latency_median_s": 1.148, "latency_p90_s": 3.587, "rtf_median": 0.1119, "cost_usd_total": 0.1581, "cost_usd_per_hour": 0.539, "audio_minutes": 17.6}, "sarvam-saaras-v4|gu": {"samples": 100, "wer": 0.1489, "cer": 0.0381, "wer_script_agnostic": 0.1489, "wer_median": 0.1154, "self_ref_rate": 0.0, "latency_median_s": 0.588, "latency_p90_s": 2.896, "rtf_median": 0.0621, "cost_usd_total": 0.1594, "cost_usd_per_hour": 0.539, "audio_minutes": 17.7}, "whisper|gu": {"samples": 100, "wer": 1.0482, "cer": 0.7559, "wer_script_agnostic": 1.0477, "wer_median": 1.0625, "self_ref_rate": 0.0, "latency_median_s": 5.679, "latency_p90_s": 17.361, "rtf_median": 0.5658, "cost_usd_total": 0.1065, "cost_usd_per_hour": 0.36, "audio_minutes": 17.7}, "gemini|gu": {"samples": 50, "wer": 0.173, "cer": 0.0429, "wer_script_agnostic": 0.173, "wer_median": 0.1429, "self_ref_rate": 0.0, "latency_median_s": 2.962, "latency_p90_s": 3.945, "rtf_median": 0.3074, "cost_usd_total": 0.0729, "cost_usd_per_hour": 0.4738, "audio_minutes": 9.2}, "voxtral|gu": {"samples": 50, "wer": 1.066, "cer": 0.8922, "wer_script_agnostic": 1.066, "wer_median": 1.1, "self_ref_rate": 0.0, "latency_median_s": 1.302, "latency_p90_s": 2.171, "rtf_median": 0.1158, "cost_usd_total": 0.0535, "cost_usd_per_hour": 0.3479, "audio_minutes": 9.2}, "gpt-audio|gu": {"samples": 36, "wer": 0.369, "cer": 0.1946, "wer_script_agnostic": 0.369, "wer_median": 0.2667, "self_ref_rate": 0.0, "latency_median_s": 1.738, "latency_p90_s": 2.603, "rtf_median": 0.1859, "cost_usd_total": 0.153, "cost_usd_per_hour": 1.4026, "audio_minutes": 6.5}, "sarvam|hi": {"samples": 3145, "wer": 0.2621, "cer": 0.1421, "wer_script_agnostic": 0.2434, "wer_median": 0.25, "self_ref_rate": 0.0242, "latency_median_s": 1.15, "latency_p90_s": 3.856, "rtf_median": 0.1197, "cost_usd_total": 4.8137, "cost_usd_per_hour": 0.5389, "audio_minutes": 535.9}, "sarvam-saaras|hi": {"samples": 3140, "wer": 0.2398, "cer": 0.1246, "wer_script_agnostic": 0.2212, "wer_median": 0.2222, "self_ref_rate": 0.0, "latency_median_s": 1.073, "latency_p90_s": 3.764, "rtf_median": 0.1114, "cost_usd_total": 4.8058, "cost_usd_per_hour": 0.5389, "audio_minutes": 535.0}, "sarvam-saaras-v4|hi": {"samples": 3145, "wer": 0.231, "cer": 0.1235, "wer_script_agnostic": 0.2139, "wer_median": 0.2121, "self_ref_rate": 0.0, "latency_median_s": 0.584, "latency_p90_s": 2.894, "rtf_median": 0.0636, "cost_usd_total": 4.8137, "cost_usd_per_hour": 0.5389, "audio_minutes": 535.9}, "whisper|hi": {"samples": 3141, "wer": 0.5625, "cer": 0.3245, "wer_script_agnostic": 0.5313, "wer_median": 0.5714, "self_ref_rate": 0.0019, "latency_median_s": 2.657, "latency_p90_s": 6.067, "rtf_median": 0.2755, "cost_usd_total": 3.2183, "cost_usd_per_hour": 0.36, "audio_minutes": 536.4}, "gemini|hi": {"samples": 282, "wer": 0.2157, "cer": 0.169, "wer_script_agnostic": 0.1995, "wer_median": 0.1538, "self_ref_rate": 0.1773, "latency_median_s": 3.459, "latency_p90_s": 4.779, "rtf_median": 0.2356, "cost_usd_total": 0.5321, "cost_usd_per_hour": 0.4301, "audio_minutes": 74.2}, "voxtral|hi": {"samples": 282, "wer": 0.917, "cer": 0.7506, "wer_script_agnostic": 0.9009, "wer_median": 0.2941, "self_ref_rate": 0.0, "latency_median_s": 1.563, "latency_p90_s": 2.44, "rtf_median": 0.1049, "cost_usd_total": 0.4406, "cost_usd_per_hour": 0.3562, "audio_minutes": 74.2}, "sarvam|kn": {"samples": 98, "wer": 0.1628, "cer": 0.0335, "wer_script_agnostic": 0.1628, "wer_median": 0.1333, "self_ref_rate": 0.0, "latency_median_s": 1.115, "latency_p90_s": 3.83, "rtf_median": 0.0813, "cost_usd_total": 0.2047, "cost_usd_per_hour": 0.539, "audio_minutes": 22.8}, "sarvam-saaras|kn": {"samples": 98, "wer": 0.1482, "cer": 0.0321, "wer_script_agnostic": 0.1482, "wer_median": 0.125, "self_ref_rate": 0.0, "latency_median_s": 1.176, "latency_p90_s": 3.894, "rtf_median": 0.0889, "cost_usd_total": 0.2047, "cost_usd_per_hour": 0.539, "audio_minutes": 22.8}, "sarvam-saaras-v4|kn": {"samples": 98, "wer": 0.1317, "cer": 0.0318, "wer_script_agnostic": 0.1317, "wer_median": 0.0952, "self_ref_rate": 0.0, "latency_median_s": 0.614, "latency_p90_s": 2.941, "rtf_median": 0.0454, "cost_usd_total": 0.2047, "cost_usd_per_hour": 0.539, "audio_minutes": 22.8}, "whisper|kn": {"samples": 100, "wer": 0.7632, "cer": 0.305, "wer_script_agnostic": 0.7632, "wer_median": 0.75, "self_ref_rate": 0.0, "latency_median_s": 4.079, "latency_p90_s": 10.437, "rtf_median": 0.3044, "cost_usd_total": 0.1431, "cost_usd_per_hour": 0.36, "audio_minutes": 23.8}, "gemini|kn": {"samples": 50, "wer": 0.1839, "cer": 0.0363, "wer_script_agnostic": 0.1839, "wer_median": 0.1429, "self_ref_rate": 0.0, "latency_median_s": 3.641, "latency_p90_s": 4.818, "rtf_median": 0.2622, "cost_usd_total": 0.0823, "cost_usd_per_hour": 0.4303, "audio_minutes": 11.5}, "voxtral|kn": {"samples": 50, "wer": 0.9205, "cer": 0.5848, "wer_script_agnostic": 0.9205, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.339, "latency_p90_s": 2.134, "rtf_median": 0.0996, "cost_usd_total": 0.0671, "cost_usd_per_hour": 0.3509, "audio_minutes": 11.5}, "sarvam|ml": {"samples": 99, "wer": 0.2157, "cer": 0.0567, "wer_script_agnostic": 0.2157, "wer_median": 0.1905, "self_ref_rate": 0.0, "latency_median_s": 1.18, "latency_p90_s": 3.938, "rtf_median": 0.0851, "cost_usd_total": 0.2311, "cost_usd_per_hour": 0.539, "audio_minutes": 25.7}, "sarvam-saaras|ml": {"samples": 99, "wer": 0.191, "cer": 0.0412, "wer_script_agnostic": 0.191, "wer_median": 0.1667, "self_ref_rate": 0.0, "latency_median_s": 1.19, "latency_p90_s": 4.016, "rtf_median": 0.0828, "cost_usd_total": 0.2311, "cost_usd_per_hour": 0.539, "audio_minutes": 25.7}, "sarvam-saaras-v4|ml": {"samples": 99, "wer": 0.1689, "cer": 0.0369, "wer_script_agnostic": 0.1689, "wer_median": 0.1538, "self_ref_rate": 0.0, "latency_median_s": 0.617, "latency_p90_s": 2.99, "rtf_median": 0.0465, "cost_usd_total": 0.2311, "cost_usd_per_hour": 0.539, "audio_minutes": 25.7}, "whisper|ml": {"samples": 100, "wer": 1.0741, "cer": 0.9517, "wer_script_agnostic": 1.0741, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 6.432, "latency_p90_s": 11.564, "rtf_median": 0.4426, "cost_usd_total": 0.1576, "cost_usd_per_hour": 0.36, "audio_minutes": 26.3}, "gemini|ml": {"samples": 50, "wer": 0.1967, "cer": 0.0407, "wer_script_agnostic": 0.1967, "wer_median": 0.1538, "self_ref_rate": 0.0, "latency_median_s": 3.664, "latency_p90_s": 5.659, "rtf_median": 0.2564, "cost_usd_total": 0.0914, "cost_usd_per_hour": 0.412, "audio_minutes": 13.3}, "voxtral|ml": {"samples": 50, "wer": 0.9949, "cer": 0.6078, "wer_script_agnostic": 0.9949, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.36, "latency_p90_s": 1.955, "rtf_median": 0.0896, "cost_usd_total": 0.078, "cost_usd_per_hour": 0.3517, "audio_minutes": 13.3}, "sarvam|mr": {"samples": 99, "wer": 0.1919, "cer": 0.0564, "wer_script_agnostic": 0.174, "wer_median": 0.1739, "self_ref_rate": 0.0, "latency_median_s": 1.181, "latency_p90_s": 3.868, "rtf_median": 0.09, "cost_usd_total": 0.2113, "cost_usd_per_hour": 0.539, "audio_minutes": 23.5}, "sarvam-saaras|mr": {"samples": 98, "wer": 0.1824, "cer": 0.0549, "wer_script_agnostic": 0.1658, "wer_median": 0.1667, "self_ref_rate": 0.0, "latency_median_s": 1.124, "latency_p90_s": 3.781, "rtf_median": 0.0914, "cost_usd_total": 0.2097, "cost_usd_per_hour": 0.539, "audio_minutes": 23.3}, "sarvam-saaras-v4|mr": {"samples": 99, "wer": 0.1477, "cer": 0.0388, "wer_script_agnostic": 0.1308, "wer_median": 0.1333, "self_ref_rate": 0.0, "latency_median_s": 0.634, "latency_p90_s": 2.905, "rtf_median": 0.048, "cost_usd_total": 0.2113, "cost_usd_per_hour": 0.539, "audio_minutes": 23.5}, "whisper|mr": {"samples": 100, "wer": 0.84, "cer": 0.3035, "wer_script_agnostic": 0.7529, "wer_median": 0.8462, "self_ref_rate": 0.0, "latency_median_s": 2.964, "latency_p90_s": 5.437, "rtf_median": 0.2132, "cost_usd_total": 0.1442, "cost_usd_per_hour": 0.36, "audio_minutes": 24.0}, "gemini|mr": {"samples": 50, "wer": 0.1786, "cer": 0.0447, "wer_script_agnostic": 0.1602, "wer_median": 0.1778, "self_ref_rate": 0.0, "latency_median_s": 2.983, "latency_p90_s": 4.434, "rtf_median": 0.2387, "cost_usd_total": 0.0793, "cost_usd_per_hour": 0.396, "audio_minutes": 12.0}, "voxtral|mr": {"samples": 50, "wer": 0.9214, "cer": 0.4107, "wer_script_agnostic": 0.8724, "wer_median": 0.7619, "self_ref_rate": 0.0, "latency_median_s": 1.444, "latency_p90_s": 2.756, "rtf_median": 0.1035, "cost_usd_total": 0.0694, "cost_usd_per_hour": 0.3464, "audio_minutes": 12.0}, "sarvam|or": {"samples": 99, "wer": 0.2165, "cer": 0.0609, "wer_script_agnostic": 0.2165, "wer_median": 0.2, "self_ref_rate": 0.0, "latency_median_s": 1.121, "latency_p90_s": 4.025, "rtf_median": 0.1041, "cost_usd_total": 0.1699, "cost_usd_per_hour": 0.539, "audio_minutes": 18.9}, "sarvam-saaras|or": {"samples": 98, "wer": 0.1866, "cer": 0.0483, "wer_script_agnostic": 0.1866, "wer_median": 0.1786, "self_ref_rate": 0.0, "latency_median_s": 1.092, "latency_p90_s": 3.544, "rtf_median": 0.1078, "cost_usd_total": 0.1687, "cost_usd_per_hour": 0.539, "audio_minutes": 18.8}, "sarvam-saaras-v4|or": {"samples": 99, "wer": 0.1534, "cer": 0.0395, "wer_script_agnostic": 0.1534, "wer_median": 0.15, "self_ref_rate": 0.0, "latency_median_s": 0.562, "latency_p90_s": 2.915, "rtf_median": 0.0601, "cost_usd_total": 0.1699, "cost_usd_per_hour": 0.539, "audio_minutes": 18.9}, "whisper|or": {"samples": 100, "wer": 1.182, "cer": 0.9901, "wer_script_agnostic": 1.182, "wer_median": 1.1333, "self_ref_rate": 0.0, "latency_median_s": 4.983, "latency_p90_s": 14.573, "rtf_median": 0.4701, "cost_usd_total": 0.1168, "cost_usd_per_hour": 0.36, "audio_minutes": 19.5}, "gemini|or": {"samples": 50, "wer": 0.2171, "cer": 0.0514, "wer_script_agnostic": 0.2171, "wer_median": 0.1818, "self_ref_rate": 0.0, "latency_median_s": 3.38, "latency_p90_s": 4.814, "rtf_median": 0.3154, "cost_usd_total": 0.0937, "cost_usd_per_hour": 0.5751, "audio_minutes": 9.8}, "voxtral|or": {"samples": 50, "wer": 1.2119, "cer": 0.9621, "wer_script_agnostic": 1.2119, "wer_median": 1.2143, "self_ref_rate": 0.0, "latency_median_s": 1.34, "latency_p90_s": 1.672, "rtf_median": 0.1133, "cost_usd_total": 0.0569, "cost_usd_per_hour": 0.3491, "audio_minutes": 9.8}, "sarvam|pa": {"samples": 100, "wer": 0.1603, "cer": 0.052, "wer_script_agnostic": 0.1603, "wer_median": 0.1379, "self_ref_rate": 0.0, "latency_median_s": 1.17, "latency_p90_s": 3.756, "rtf_median": 0.1078, "cost_usd_total": 0.1701, "cost_usd_per_hour": 0.539, "audio_minutes": 18.9}, "sarvam-saaras|pa": {"samples": 100, "wer": 0.1396, "cer": 0.0449, "wer_script_agnostic": 0.1396, "wer_median": 0.1111, "self_ref_rate": 0.0, "latency_median_s": 1.226, "latency_p90_s": 3.783, "rtf_median": 0.1141, "cost_usd_total": 0.1701, "cost_usd_per_hour": 0.539, "audio_minutes": 18.9}, "sarvam-saaras-v4|pa": {"samples": 100, "wer": 0.1107, "cer": 0.0328, "wer_script_agnostic": 0.1107, "wer_median": 0.1, "self_ref_rate": 0.0, "latency_median_s": 0.594, "latency_p90_s": 2.929, "rtf_median": 0.0648, "cost_usd_total": 0.1701, "cost_usd_per_hour": 0.539, "audio_minutes": 18.9}, "whisper|pa": {"samples": 100, "wer": 1.0781, "cer": 0.8555, "wer_script_agnostic": 1.0781, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 4.167, "latency_p90_s": 9.947, "rtf_median": 0.4155, "cost_usd_total": 0.1136, "cost_usd_per_hour": 0.36, "audio_minutes": 18.9}, "gemini|pa": {"samples": 50, "wer": 0.1497, "cer": 0.0488, "wer_script_agnostic": 0.1497, "wer_median": 0.125, "self_ref_rate": 0.0, "latency_median_s": 3.258, "latency_p90_s": 4.229, "rtf_median": 0.3078, "cost_usd_total": 0.08, "cost_usd_per_hour": 0.5113, "audio_minutes": 9.4}, "voxtral|pa": {"samples": 50, "wer": 1.0162, "cer": 0.889, "wer_script_agnostic": 1.0162, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.217, "latency_p90_s": 1.724, "rtf_median": 0.1119, "cost_usd_total": 0.0542, "cost_usd_per_hour": 0.3465, "audio_minutes": 9.4}, "sarvam|ta": {"samples": 97, "wer": 0.2684, "cer": 0.1104, "wer_script_agnostic": 0.2684, "wer_median": 0.2222, "self_ref_rate": 0.0, "latency_median_s": 1.078, "latency_p90_s": 3.749, "rtf_median": 0.0949, "cost_usd_total": 0.1855, "cost_usd_per_hour": 0.539, "audio_minutes": 20.6}, "sarvam-saaras|ta": {"samples": 97, "wer": 0.2271, "cer": 0.0802, "wer_script_agnostic": 0.2271, "wer_median": 0.2, "self_ref_rate": 0.0, "latency_median_s": 1.127, "latency_p90_s": 3.914, "rtf_median": 0.0954, "cost_usd_total": 0.1855, "cost_usd_per_hour": 0.539, "audio_minutes": 20.6}, "sarvam-saaras-v4|ta": {"samples": 97, "wer": 0.2195, "cer": 0.0772, "wer_script_agnostic": 0.2195, "wer_median": 0.1818, "self_ref_rate": 0.0, "latency_median_s": 0.601, "latency_p90_s": 2.917, "rtf_median": 0.0523, "cost_usd_total": 0.1855, "cost_usd_per_hour": 0.539, "audio_minutes": 20.6}, "whisper|ta": {"samples": 100, "wer": 0.5933, "cer": 0.2094, "wer_script_agnostic": 0.5933, "wer_median": 0.5455, "self_ref_rate": 0.0, "latency_median_s": 3.005, "latency_p90_s": 10.627, "rtf_median": 0.2865, "cost_usd_total": 0.1378, "cost_usd_per_hour": 0.36, "audio_minutes": 23.0}, "gemini|ta": {"samples": 50, "wer": 0.2485, "cer": 0.1031, "wer_script_agnostic": 0.2485, "wer_median": 0.1538, "self_ref_rate": 0.0, "latency_median_s": 3.4, "latency_p90_s": 4.172, "rtf_median": 0.2924, "cost_usd_total": 0.0774, "cost_usd_per_hour": 0.4101, "audio_minutes": 11.3}, "voxtral|ta": {"samples": 50, "wer": 0.7698, "cer": 0.4167, "wer_script_agnostic": 0.7698, "wer_median": 0.6667, "self_ref_rate": 0.0, "latency_median_s": 1.368, "latency_p90_s": 1.785, "rtf_median": 0.1145, "cost_usd_total": 0.0668, "cost_usd_per_hour": 0.3539, "audio_minutes": 11.3}, "sarvam|te": {"samples": 99, "wer": 0.2195, "cer": 0.0663, "wer_script_agnostic": 0.2195, "wer_median": 0.2, "self_ref_rate": 0.0, "latency_median_s": 1.041, "latency_p90_s": 4.062, "rtf_median": 0.1032, "cost_usd_total": 0.1597, "cost_usd_per_hour": 0.539, "audio_minutes": 17.8}, "sarvam-saaras|te": {"samples": 99, "wer": 0.1971, "cer": 0.0576, "wer_script_agnostic": 0.1971, "wer_median": 0.1786, "self_ref_rate": 0.0, "latency_median_s": 1.075, "latency_p90_s": 3.708, "rtf_median": 0.1079, "cost_usd_total": 0.1597, "cost_usd_per_hour": 0.539, "audio_minutes": 17.8}, "sarvam-saaras-v4|te": {"samples": 99, "wer": 0.1607, "cer": 0.04, "wer_script_agnostic": 0.1607, "wer_median": 0.1176, "self_ref_rate": 0.0, "latency_median_s": 0.585, "latency_p90_s": 2.964, "rtf_median": 0.0606, "cost_usd_total": 0.1597, "cost_usd_per_hour": 0.539, "audio_minutes": 17.8}, "whisper|te": {"samples": 100, "wer": 1.0532, "cer": 0.7188, "wer_script_agnostic": 1.0532, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 6.116, "latency_p90_s": 14.765, "rtf_median": 0.6283, "cost_usd_total": 0.1098, "cost_usd_per_hour": 0.36, "audio_minutes": 18.3}, "gemini|te": {"samples": 50, "wer": 0.1961, "cer": 0.0436, "wer_script_agnostic": 0.1961, "wer_median": 0.1579, "self_ref_rate": 0.0, "latency_median_s": 3.185, "latency_p90_s": 4.679, "rtf_median": 0.3318, "cost_usd_total": 0.0688, "cost_usd_per_hour": 0.4586, "audio_minutes": 9.0}, "voxtral|te": {"samples": 50, "wer": 2.0838, "cer": 1.4725, "wer_script_agnostic": 2.0838, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 1.242, "latency_p90_s": 2.254, "rtf_median": 0.1269, "cost_usd_total": 0.0537, "cost_usd_per_hour": 0.3579, "audio_minutes": 9.0}, "sarvam|ur": {"samples": 100, "wer": 1.0154, "cer": 0.8743, "wer_script_agnostic": 1.0154, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.238, "latency_p90_s": 3.761, "rtf_median": 0.1315, "cost_usd_total": 0.1426, "cost_usd_per_hour": 0.539, "audio_minutes": 15.9}, "sarvam-saaras|ur": {"samples": 100, "wer": 1.007, "cer": 0.8674, "wer_script_agnostic": 1.007, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.183, "latency_p90_s": 3.791, "rtf_median": 0.1284, "cost_usd_total": 0.1426, "cost_usd_per_hour": 0.539, "audio_minutes": 15.9}, "sarvam-saaras-v4|ur": {"samples": 100, "wer": 0.5137, "cer": 0.4101, "wer_script_agnostic": 0.5137, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 0.592, "latency_p90_s": 2.964, "rtf_median": 0.0688, "cost_usd_total": 0.1426, "cost_usd_per_hour": 0.539, "audio_minutes": 15.9}, "whisper|ur": {"samples": 100, "wer": 0.2407, "cer": 0.083, "wer_script_agnostic": 0.2407, "wer_median": 0.2424, "self_ref_rate": 0.0, "latency_median_s": 2.036, "latency_p90_s": 3.343, "rtf_median": 0.2091, "cost_usd_total": 0.0953, "cost_usd_per_hour": 0.3601, "audio_minutes": 15.9}, "gemini|ur": {"samples": 50, "wer": 0.1359, "cer": 0.0428, "wer_script_agnostic": 0.1359, "wer_median": 0.1429, "self_ref_rate": 0.0, "latency_median_s": 3.3, "latency_p90_s": 4.459, "rtf_median": 0.3477, "cost_usd_total": 0.0639, "cost_usd_per_hour": 0.4666, "audio_minutes": 8.2}, "voxtral|ur": {"samples": 50, "wer": 0.8843, "cer": 0.8048, "wer_script_agnostic": 0.8843, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.255, "latency_p90_s": 1.699, "rtf_median": 0.1235, "cost_usd_total": 0.0476, "cost_usd_per_hour": 0.3479, "audio_minutes": 8.2}, "sarvam|hi-en": {"samples": 3126, "wer": 0.5476, "cer": 0.5014, "wer_script_agnostic": 0.5282, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 1.014, "latency_p90_s": 3.655, "rtf_median": 0.2099, "cost_usd_total": 2.7663, "cost_usd_per_hour": 0.539, "audio_minutes": 307.9}, "sarvam-saaras|hi-en": {"samples": 3111, "wer": 0.5416, "cer": 0.5041, "wer_script_agnostic": 0.5217, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 1.02, "latency_p90_s": 3.665, "rtf_median": 0.2117, "cost_usd_total": 2.7543, "cost_usd_per_hour": 0.539, "audio_minutes": 306.6}, "sarvam-saaras-v4|hi-en": {"samples": 3127, "wer": 0.5061, "cer": 0.4699, "wer_script_agnostic": 0.4899, "wer_median": 0.5, "self_ref_rate": 0.0, "latency_median_s": 0.526, "latency_p90_s": 2.83, "rtf_median": 0.1132, "cost_usd_total": 2.7678, "cost_usd_per_hour": 0.539, "audio_minutes": 308.1}, "whisper|hi-en": {"samples": 3129, "wer": 0.5746, "cer": 0.4765, "wer_script_agnostic": 0.5514, "wer_median": 0.5417, "self_ref_rate": 0.0, "latency_median_s": 1.942, "latency_p90_s": 3.56, "rtf_median": 0.3635, "cost_usd_total": 1.8626, "cost_usd_per_hour": 0.3601, "audio_minutes": 310.4}, "gemini|hi-en": {"samples": 99, "wer": 0.4906, "cer": 0.4781, "wer_script_agnostic": 0.4795, "wer_median": 0.5294, "self_ref_rate": 0.0, "latency_median_s": 2.83, "latency_p90_s": 3.887, "rtf_median": 0.587, "cost_usd_total": 0.076, "cost_usd_per_hour": 0.4558, "audio_minutes": 10.0}, "voxtral|hi-en": {"samples": 100, "wer": 9.2611, "cer": 9.6549, "wer_script_agnostic": 9.2543, "wer_median": 1.125, "self_ref_rate": 0.0, "latency_median_s": 1.237, "latency_p90_s": 3.443, "rtf_median": 0.2287, "cost_usd_total": 0.0663, "cost_usd_per_hour": 0.3949, "audio_minutes": 10.1}, "gpt-audio|hi-en": {"samples": 100, "wer": 1.0323, "cer": 0.9865, "wer_script_agnostic": 1.0298, "wer_median": 1.0, "self_ref_rate": 0.0, "latency_median_s": 1.59, "latency_p90_s": 3.037, "rtf_median": 0.3236, "cost_usd_total": 0.244, "cost_usd_per_hour": 1.4531, "audio_minutes": 10.1}, "gpt-audio|hi": {"samples": 232, "wer": 0.3264, "cer": 0.2602, "wer_script_agnostic": 0.3121, "wer_median": 0.2949, "self_ref_rate": 0.0, "latency_median_s": 2.289, "latency_p90_s": 3.325, "rtf_median": 0.1357, "cost_usd_total": 1.4679, "cost_usd_per_hour": 1.3598, "audio_minutes": 64.8}}, "silver_caveat_providers": ["gemini", "sarvam", "whisper"], "row_count": 31755, "sample_count": 7385};
const P = ["sarvam-saaras-v4","sarvam-saaras","sarvam","whisper","gemini","voxtral","gpt-audio"];
const PCOLOR = {"sarvam-saaras-v4":"#6DC3AD","sarvam-saaras":"#42AD94",sarvam:"#3EA0B2",whisper:"#8670AA",gemini:"#E89A48",voxtral:"#C96F79","gpt-audio":"#619C4E"};
const PLABEL = {"sarvam-saaras-v4":"Sarvam saaras v4","sarvam-saaras":"Sarvam saaras v3",sarvam:"Sarvam saarika v2.5",whisper:"OpenAI Whisper",gemini:"Gemini 3.5 Flash",voxtral:"Mistral Voxtral","gpt-audio":"OpenAI gpt-audio"};
const PSUB = {"sarvam-saaras-v4":"Sarvam's latest, added Sep 2026","sarvam-saaras":"what Logatra runs in production",sarvam:"previous Sarvam generation",whisper:"whisper-1 API",gemini:"via OpenRouter, minimal reasoning",voxtral:"voxtral-small-24b","gpt-audio":"OpenAI audio chat model"};
const DSLABEL = {fleurs:"Read speech · FLEURS",mucs_hinglish:"Hinglish · MUCS",gramvaani:"Phone calls · Gramvaani",heldout_yt:"Real-world · fresh YouTube"};
const cssVar=n=>getComputedStyle(document.documentElement).getPropertyValue(n).trim();
function applyChartDefaults(){Chart.defaults.color=cssVar("--chart-text");Chart.defaults.borderColor=cssVar("--chart-grid")}
applyChartDefaults();Chart.defaults.font.family="'Plus Jakarta Sans',sans-serif";Chart.defaults.font.size=11.5;
const CASES = window.LOGATRA_CASES||[];
const g=(o,k)=>o[k]||null;
const acc=w=>w==null?null:Math.max(0,(1-w)*100);
const reduced=matchMedia("(prefers-reduced-motion: reduce)").matches;
// reveal on scroll
const io=new IntersectionObserver(es=>es.forEach(e=>{if(e.isIntersecting){e.target.classList.add("in");io.unobserve(e.target)}}),{threshold:.06});
document.querySelectorAll(".reveal").forEach(el=>io.observe(el));
// stat strip with count-up
(function(){
const langs=new Set(CASES.map(c=>c.langName));
const mins=Math.round(CASES.reduce((a,c)=>a+1,0)&&DATA?Object.values(DATA.by_provider).reduce((m,v)=>Math.max(m,v.audio_minutes),0):0);
const items=[[CASES.length,"test cases you can inspect"],[langs.size,"languages & test types"],[P.length,"engines compared"],[CASES.filter(c=>c.real).length,"fresh 2026 real-world clips"],[mins,"minutes of audio per engine"]];
const el=document.getElementById("statstrip");
items.forEach(([n,s])=>{const d=document.createElement("div");d.innerHTML=`<b data-n="${n}">0</b><span>${s}</span>`;el.appendChild(d)});
el.querySelectorAll("b").forEach(b=>{
const target=+b.dataset.n;if(reduced){b.textContent=target.toLocaleString();return}
const t0=performance.now();
(function tick(t){const k=Math.min(1,(t-t0)/900);b.textContent=Math.round(target*(1-Math.pow(1-k,3))).toLocaleString();if(k<1)requestAnimationFrame(tick)})(t0);
});
})();
// leaderboard
const HINTS={heldout:"Fresh Creative-Commons audio published Jan–Jun 2026, after every model's training cutoff. The honest number.",
public:"FLEURS + MUCS + Gramvaani. Public data that models have likely seen in training; a flattering baseline."};
function renderBoard(split){
document.getElementById("splitHint").textContent=HINTS[split]+(DATA.comparable_counts[split]?` Same ${DATA.comparable_counts[split].toLocaleString()} clips for every engine.`:"");
const el=document.getElementById("board");el.innerHTML="";
const src=DATA.by_provider_comparable&&Object.keys(DATA.by_provider_comparable).length?DATA.by_provider_comparable:DATA.by_provider;
const rows=P.map(p=>({p,d:g(src,p+"|"+split)})).filter(r=>r.d&&r.d.samples>0)
.map(r=>({...r,acc:acc(r.d.wer_script_agnostic)})).sort((a,b)=>b.acc-a.acc);
rows.forEach((r,i)=>{
const div=document.createElement("div");div.className="lbrow"+(i===0?" first":"");
const flags=[];
if(split==="heldout"&&DATA.silver_caveat_providers.includes(r.p))flags.push('<span class="flag note" title="helped create the reference transcripts; its score is an upper bound">upper bound</span>');
if(r.p==="voxtral")flags.push('<span class="flag" title="frequently repeats phrases endlessly and translates Hindi to English">loops & translates</span>');
div.innerHTML=`<div class="rank">${i===0?"①":i+1}</div>
<div class="who"><span class="dot" style="background:${PCOLOR[r.p]}"></span>${PLABEL[r.p]}${flags.join("")}<small>${PSUB[r.p]}</small></div>
<div class="accbar"><div class="bar"><div class="fill" style="background:${PCOLOR[r.p]}"></div></div><b>${r.acc.toFixed(1)}%</b></div>
<div class="cell" title="median seconds to process one clip">${r.d.latency_median_s==null?"–":r.d.latency_median_s.toFixed(1)+" s/clip"}</div>
<div class="cell" title="cost per hour of audio">${r.d.cost_usd_per_hour==null?"–":"$"+r.d.cost_usd_per_hour.toFixed(2)+"/hr"}</div>`;
el.appendChild(div);
requestAnimationFrame(()=>requestAnimationFrame(()=>{div.querySelector(".fill").style.width=(reduced?r.acc:r.acc)+"%"}));
});
}
document.querySelectorAll("#splitToggle button").forEach(b=>b.addEventListener("click",()=>{
document.querySelectorAll("#splitToggle button").forEach(x=>x.classList.remove("on"));
b.classList.add("on");renderBoard(b.dataset.split)}));
renderBoard("heldout");
// language x engine matrix from per-case data
function renderMatrix(){
const rowKey=c=>c.ds==="mucs_hinglish"?"Hinglish (conversations)":c.ds==="gramvaani"?"Hindi (phone calls)":c.real?"Real-world Hinglish (2026)":c.langName+" (read)";
const agg={};
CASES.forEach(c=>{const k=rowKey(c);agg[k]=agg[k]||{};
P.forEach(p=>{const h=c.p[p];if(!h||h.as_==null)return;(agg[k][p]=agg[k][p]||[]).push(h.as_)})});
const rows=Object.keys(agg).sort((a,b)=>a.localeCompare(b));
const mx=document.getElementById("mx");mx.innerHTML="";
mx.style.gridTemplateColumns=`200px repeat(${P.length},1fr)`;
const low=cssVar("--mx-low").split(",").map(Number),lowText=cssVar("--mx-text-low");
mx.insertAdjacentHTML("beforeend",`<div class="colh rowh">test type</div>`+P.map(p=>`<div class="colh"><span class="dot" style="background:${PCOLOR[p]}"></span>${PLABEL[p].replace("Sarvam ","").replace("OpenAI ","").replace("Mistral ","")}</div>`).join(""));
const shade=v=>{ // 0..100 -> sand -> kai
if(v==null)return["transparent","var(--ink400)"];
const t=Math.max(0,Math.min(1,v/100));
const mix=(a,b)=>Math.round(a+(b-a)*t);
return[`rgb(${mix(low[0],31)},${mix(low[1],134)},${mix(low[2],154)})`, t>.55?"#F2EBDD":lowText];
};
rows.forEach(rk=>{
mx.insertAdjacentHTML("beforeend",`<div class="rowh">${rk}</div>`+P.map(p=>{
const arr=agg[rk][p];const v=arr?arr.reduce((a,x)=>a+x,0)/arr.length:null;
const[bg,fg]=shade(v);
return`<div class="v" style="background:${bg};color:${fg}">${v==null?"–":v.toFixed(0)+"%"}</div>`}).join(""));
});
}
renderMatrix();
// charts
let CHARTS=[];
function renderCharts(){
CHARTS.forEach(c=>c.destroy());CHARTS=[];applyChartDefaults();const grid=cssVar("--chart-grid");
const ds=Object.keys(DSLABEL).filter(d=>P.some(p=>g(DATA.by_provider_dataset,p+"|"+d)));
CHARTS.push(new Chart(cDataset,{type:"bar",data:{labels:ds.map(d=>DSLABEL[d]),
datasets:P.map(p=>({label:PLABEL[p],backgroundColor:PCOLOR[p],borderRadius:3,
data:ds.map(d=>{const v=g(DATA.by_provider_dataset,p+"|"+d);return v?acc(v.wer_script_agnostic):null})})).filter(s=>s.data.some(v=>v!=null))},
options:{maintainAspectRatio:false,scales:{y:{min:0,max:100,title:{display:true,text:"accuracy %"},grid:{color:grid}},x:{grid:{display:false}}},plugins:{legend:{labels:{boxWidth:10,boxHeight:10}}}}}));
const pts=P.map(p=>{const v=g(DATA.by_provider_comparable,p+"|public")||g(DATA.by_provider,p+"|public");if(!v||!v.latency_median_s)return null;
return{label:PLABEL[p],backgroundColor:PCOLOR[p]+"CC",borderColor:PCOLOR[p],
data:[{x:v.latency_median_s,y:acc(v.wer_script_agnostic),r:Math.max(6,Math.sqrt(v.cost_usd_per_hour||.2)*9)}]}}).filter(Boolean);
CHARTS.push(new Chart(cScatter,{type:"bubble",data:{datasets:pts},options:{maintainAspectRatio:false,
scales:{x:{title:{display:true,text:"median seconds per clip"},grid:{color:grid}},y:{min:0,max:100,title:{display:true,text:"accuracy %"},grid:{color:grid}}},
plugins:{legend:{labels:{boxWidth:10,boxHeight:10}},tooltip:{callbacks:{label:c=>`${c.dataset.label}: ${c.raw.y.toFixed(1)}% acc, ${c.raw.x.toFixed(1)}s/clip`}}}}}));
const provs=P.filter(p=>g(DATA.by_provider,p+"|public"));
CHARTS.push(new Chart(cCost,{type:"bar",data:{labels:provs.map(p=>PLABEL[p].replace("Sarvam ","").replace("OpenAI ","").replace("Mistral ","")),
datasets:[{label:"$ per audio hour",data:provs.map(p=>g(DATA.by_provider,p+"|public").cost_usd_per_hour),backgroundColor:provs.map(p=>PCOLOR[p]),borderRadius:3}]},
options:{maintainAspectRatio:false,scales:{y:{title:{display:true,text:"$ / hr"},grid:{color:grid}},x:{grid:{display:false},ticks:{maxRotation:30}}},plugins:{legend:{display:false}}}}));
CHARTS.push(new Chart(cRtf,{type:"bar",data:{labels:provs.map(p=>PLABEL[p].replace("Sarvam ","").replace("OpenAI ","").replace("Mistral ","")),
datasets:[{label:"real-time factor",data:provs.map(p=>g(DATA.by_provider,p+"|public").rtf_median),backgroundColor:provs.map(p=>PCOLOR[p]+"99"),borderRadius:3}]},
options:{maintainAspectRatio:false,scales:{y:{title:{display:true,text:"processing s / audio s"},grid:{color:grid}},x:{grid:{display:false},ticks:{maxRotation:30}}},plugins:{legend:{display:false}}}}));
}
renderCharts();
// theme toggle (shares the nemukai-theme key with nemukai.com)
(function(){
const SUN='<circle cx="12" cy="12" r="5" fill="currentColor" stroke="none"/><path d="M12 1v2M12 21v2M4.22 4.22l1.42 1.42M18.36 18.36l1.42 1.42M1 12h2M21 12h2M4.22 19.78l1.42-1.42M18.36 5.64l1.42-1.42"/>';
const MOON='<path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"/>';
const btn=document.getElementById("themeToggle"),icon=document.getElementById("themeIcon"),label=document.getElementById("themeLabel");
function paint(){const dark=document.documentElement.getAttribute("data-theme")==="dark";
icon.innerHTML=dark?SUN:MOON;label.textContent=dark?"Sunny":"Shade";btn.setAttribute("aria-label",dark?"Switch to Sunny mode":"Switch to Shade mode")}
paint();
btn.addEventListener("click",()=>{const next=document.documentElement.getAttribute("data-theme")==="dark"?"light":"dark";
document.documentElement.setAttribute("data-theme",next);localStorage.setItem("nemukai-theme",next);
paint();renderMatrix();renderCharts()});
})();
// ---------- explorer ----------
const DIGITS={"०":"0","१":"1","२":"2","३":"3","४":"4","५":"5","६":"6","७":"7","८":"8","९":"9"};
function norm(t){return t.normalize("NFC").toLowerCase()
.replace(/[०-९]/g,d=>DIGITS[d])
.replace(/[.,!?;:"'()\[\]{}«»“”‘’–—…।॥۔،؟]/g," ")
.replace(/\s+/g," ").trim()}
function tokens(t){const n=norm(t);return n?n.split(" "):[]}
// LCS-based word diff; returns ops over hyp tokens + deletions
function diff(refT,hypT){
const n=refT.length,m=hypT.length;
if(n*m>40000)return null; // too long (loops) -> no diff
const dp=Array.from({length:n+1},()=>new Uint16Array(m+1));
for(let i=n-1;i>=0;i--)for(let j=m-1;j>=0;j--)
dp[i][j]=refT[i]===hypT[j]?dp[i+1][j+1]+1:Math.max(dp[i+1][j],dp[i][j+1]);
const ops=[];let i=0,j=0;
while(i<n&&j<m){
if(refT[i]===hypT[j]){ops.push(["eq",hypT[j]]);i++;j++}
else if(dp[i+1][j]>=dp[i][j+1]){ops.push(["del",refT[i]]);i++}
else{ops.push(["ins",hypT[j]]);j++}
}
while(i<n)ops.push(["del",refT[i++]]);
while(j<m)ops.push(["ins",hypT[j++]]);
// merge adjacent del+ins into substitutions for nicer display
const out=[];for(const op of ops){
const last=out[out.length-1];
if(op[0]==="ins"&&last&&last[0]==="del"){out[out.length-1]=["sub",op[1]];continue}
out.push(op)}
return out;
}
function esc(s){return s.replace(/&/g,"&amp;").replace(/</g,"&lt;")}
function renderDiff(ref,hyp){
const d=diff(tokens(ref),tokens(hyp));
if(!d)return esc(hyp);
return d.map(([op,w])=>op==="eq"?esc(w):op==="sub"?`<span class="sub-w">${esc(w)}</span>`:op==="ins"?`<span class="ins-w">${esc(w)}</span>`:`<span class="del-w">${esc(w)}</span>`).join(" ");
}
const fDs=document.getElementById("fDs"),fLang=document.getElementById("fLang"),
fSort=document.getElementById("fSort"),fQ=document.getElementById("fQ");
[...new Set(CASES.map(c=>c.ds))].forEach(d=>fDs.insertAdjacentHTML("beforeend",`<option value="${d}">${DSLABEL[d]||d}</option>`));
[...new Set(CASES.map(c=>c.langName))].sort().forEach(l=>fLang.insertAdjacentHTML("beforeend",`<option>${l}</option>`));
const spread=c=>{const v=Object.values(c.p).map(h=>h.as_).filter(x=>x!=null);return v.length<2?0:Math.max(...v)-Math.min(...v)};
const meanA=c=>{const v=Object.values(c.p).map(h=>h.as_).filter(x=>x!=null);return v.length?v.reduce((a,x)=>a+x,0)/v.length:0};
let page=0;const PER=25;
function filtered(){
let xs=CASES;
if(fDs.value)xs=xs.filter(c=>c.ds===fDs.value);
if(fLang.value)xs=xs.filter(c=>c.langName===fLang.value);
const q=fQ.value.trim().toLowerCase();
if(q)xs=xs.filter(c=>c.ref.toLowerCase().includes(q)||Object.values(c.p).some(h=>h.t.toLowerCase().includes(q)));
const s=fSort.value;
xs=[...xs];
if(s==="disagree")xs.sort((a,b)=>spread(b)-spread(a));
else if(s==="hard")xs.sort((a,b)=>meanA(a)-meanA(b));
else if(s==="easy")xs.sort((a,b)=>meanA(b)-meanA(a));
else xs.sort((a,b)=>a.id.localeCompare(b.id));
return xs;
}
function chipColor(v){if(v==null)return"#524A3E";const t=Math.max(0,Math.min(1,v/100));
const mix=(a,b)=>Math.round(a+(b-a)*t);return`rgb(${mix(123,31)},${mix(65,134)},${mix(71,154)})`}
function renderCases(){
const xs=filtered();
const pages=Math.max(1,Math.ceil(xs.length/PER));
page=Math.min(page,pages-1);
document.getElementById("fCount").textContent=`${xs.length.toLocaleString()} cases match`;
document.getElementById("pgInfo").textContent=`page ${page+1} of ${pages}`;
document.getElementById("pgPrev").disabled=page===0;
document.getElementById("pgNext").disabled=page>=pages-1;
const root=document.getElementById("caselist");root.innerHTML="";
xs.slice(page*PER,(page+1)*PER).forEach(c=>{
const div=document.createElement("div");div.className="kcase";
const chips=P.filter(p=>c.p[p]).map(p=>`<span class="chip" title="${PLABEL[p]}" style="background:${chipColor(c.p[p].as_)}">${PLABEL[p].split(" ")[0]==="Sarvam"?(p==="sarvam"?"saarika":"saaras"):PLABEL[p].split(" ").pop().toLowerCase()} ${c.p[p].as_==null?"–":c.p[p].as_.toFixed(0)+"%"}</span>`).join("");
div.innerHTML=`<div class="khead"><span class="kid">${c.id}</span>
<span class="tag">${c.langName}</span><span class="tag ${c.real?"real":""}">${c.real?"real-world 2026":DSLABEL[c.ds].split(" ·")[0].toLowerCase()}</span>
<span class="chips">${chips}</span></div><div class="kbody"></div>`;
div.querySelector(".khead").addEventListener("click",()=>{
const open=div.classList.toggle("open");
const body=div.querySelector(".kbody");
if(open&&!body.dataset.done){
let html=`<div class="krow ref"><div class="who">ground truth${c.silver?' <span class="flag note" title="multi-model consensus reference">consensus</span>':""}</div><div class="txt">${esc(c.ref)}</div><div class="acc"></div></div>`;
P.filter(p=>c.p[p]).forEach(p=>{const h=c.p[p];
html+=`<div class="krow"><div class="who"><span class="dot" style="background:${PCOLOR[p]}"></span>${PLABEL[p]}</div>
<div class="txt">${renderDiff(c.ref,h.t)}${h.cut?'<span class="cutnote">… output truncated (engine repeated itself past this point)</span>':""}</div>
<div class="acc" title="accuracy, script-agnostic">${h.as_==null?"–":h.as_.toFixed(0)+"%"}</div></div>`});
body.innerHTML=html;body.dataset.done=1;
}
});
root.appendChild(div);
});
}
[fDs,fLang,fSort].forEach(el=>el.addEventListener("change",()=>{page=0;renderCases()}));
let qt;fQ.addEventListener("input",()=>{clearTimeout(qt);qt=setTimeout(()=>{page=0;renderCases()},250)});
document.getElementById("pgPrev").addEventListener("click",()=>{page--;renderCases()});
document.getElementById("pgNext").addEventListener("click",()=>{page++;renderCases();document.getElementById("explorer").scrollIntoView({behavior:reduced?"auto":"smooth"})});
renderCases();
document.getElementById("modelnote").textContent =
"sarvam saaras:v4 (mode=transcribe, added Sep 2026) · sarvam saaras:v3 (mode=transcribe) · sarvam saarika:v2.5 · openai whisper-1 (auto-detect fallback where hints are unsupported) · google/gemini-3.5-flash via OpenRouter at minimal reasoning · mistralai/voxtral-small-24b-2507 · openai/gpt-audio. Costs from per-call billing (OpenRouter) or list price (Sarvam ₹0.75/min, Whisper $0.006/min). Leaderboard uses only clips every engine completed.";
</script>
</body>
</html>