Hugging Face's logo

Spaces:
Grets
/
AVIP
Running

App Files Community
AVIP / index.html
Grets's picture
Grets
Update index.html
7b93b09 verified
raw
history blame contribute delete
22.3 kB
<!doctype html>
<html lang="en">
<head>
 <meta charset="utf-8" />
 <meta name="viewport" content="width=device-width, initial-scale=1" />
 <title>AVIP – Audio–Vision Interaction Probe (Benchmark)</title>
 <meta name="description" content="AVIP is a lightweight benchmark to check whether multimodal models truly use audio in videos." />
<!-- Social cards (add your own image at assets/cover.jpg) -->
 <meta property="og:title" content="AVIP Benchmark" />
 <meta property="og:description" content="Do multimodal models actually use audio in videos?" />
 <meta property="og:type" content="website" />
 <meta property="og:video" content="thumbnail" />
 <meta name="twitter:card" content="summary_large_image" />
<link rel="icon" href="assets/favicon.png" />
 <style>
 /* =====================
 THEME & BASICS
 ===================== */
 :root{
 --bg:#0b0b0c; --surface:#111216; --text:#e9e9ee; --muted:#9aa0a6; --accent:#6ee7ff; --ring:rgba(110,231,255,.35);
 --card: color-mix(in oklab, var(--surface), transparent 8%);
 --border: 1px solid rgba(255,255,255,.08);
 }
 @media (prefers-color-scheme: light){
 :root{ --bg:#fafafa; --surface:#ffffff; --text:#101114; --muted:#5f6368; --accent:#0078ff; --ring:rgba(0,120,255,.2);
 --card: color-mix(in oklab, var(--surface), transparent 4%);
 --border: 1px solid rgba(0,0,0,.08);
 }
 }
 *,*::before,*::after{ box-sizing:border-box }
 html,body{ height:100% }
 body{
 margin:0; font:16px/1.6 system-ui,-apple-system,Segoe UI,Roboto,Arial,sans-serif; color:var(--text);
 background:
 radial-gradient(1200px 800px at 10% -5%, rgba(110,231,255,.08), transparent 40%),
 radial-gradient(900px 700px at 110% 10%, rgba(110,231,255,.06), transparent 40%),
 var(--bg);
 }
 .site-header{ position:sticky; top:0; z-index:20; display:flex; align-items:center; gap:1rem; justify-content:space-between;
 padding:.8rem 1rem; border-bottom:var(--border);
 background:color-mix(in oklab, var(--surface), transparent 35%);
 backdrop-filter:saturate(1.2) blur(8px);
 }
 .logo{ font-weight:800; letter-spacing:.2px; text-decoration:none; color:var(--text); display:flex; align-items:center; gap:.55rem }
 .logo .dot{ width:.7rem; height:.7rem; border-radius:999px; background:var(--accent); box-shadow:0 0 16px var(--ring) }
 .logo span{ color:var(--accent) }
 .site-nav{ display:flex; gap:.75rem }
 .site-nav a{ color:var(--text); text-decoration:none; padding:.4rem .6rem; border-radius:.5rem }
 .site-nav a:hover{ outline:2px solid var(--ring); outline-offset:2px }
 .nav-toggle{ display:none; background:transparent; border:1px solid rgba(255,255,255,.2); color:var(--text); border-radius:.5rem; padding:.4rem .6rem }
 @media (max-width: 720px){
 .nav-toggle{ display:inline-block }
 .site-nav{ position:absolute; right:1rem; top:3.4rem; flex-direction:column; padding:.6rem; background:var(--surface); border:var(--border); border-radius:.6rem; display:none }
 .site-nav.open{ display:flex }
 }
 .hero{ text-align:center; padding:4.5rem 1rem 2rem; max-width:1060px; margin:0 auto }
 .hero h1{ font-size:clamp(2rem,3.6vw,3rem); margin:0 0 .5rem }
 .hero p{ margin:0 0 1.5rem; color:var(--muted) }
 .btn{ display:inline-block; padding:.72rem 1rem; border-radius:.6rem; background:var(--accent); color:#06141b; text-decoration:none; font-weight:700 }
 .cover{ margin:1.2rem auto 0; max-width:980px; aspect-ratio: 16/9; border-radius:.8rem; overflow:hidden; border:var(--border); background:var(--card) }
 .cover img{ width:100%; height:100%; object-fit:cover; display:block }
 .section{ padding:2.6rem 1rem; max-width:1060px; margin:0 auto }
 .section h2{ font-size:clamp(1.4rem,2.2vw,1.8rem); margin:0 0 .4rem }
 .muted{ color:var(--muted) }
 .features{ display:grid; gap:.8rem; grid-template-columns:repeat(auto-fill,minmax(230px,1fr)); margin:1rem 0 0 }
 .card{ background:var(--card); border:var(--border); border-radius:.8rem; padding:1rem }
 /* Leaderboard */
 .table-wrap{ overflow-x:auto; background:var(--card); border-radius:.8rem; border:var(--border) }
 table{ width:100%; border-collapse:collapse }
 th,td{ padding:.7rem .8rem; border-bottom:1px solid rgba(255,255,255,.06); text-align:left }
 th{ font-weight:700 }
 .pill{ display:inline-block; padding:.2rem .5rem; border-radius:.5rem; background:rgba(110,231,255,.18); color:var(--text); font-size:.8rem }
 /* Media showcase */
 .media-grid{ display:grid; grid-template-columns:2fr 1fr; gap:1rem }
 @media (max-width: 920px){ .media-grid{ grid-template-columns:1fr } }
 figure{ margin:0 }
 .media-card{ background:var(--card); border:var(--border); border-radius:.8rem; padding:.8rem }
 video{ width:100%; max-height:520px; background:#000; border-radius:.6rem }
 .toolbar{ display:flex; gap:.5rem; flex-wrap:wrap; margin:.6rem 0 }
 .toolbar button{ background:transparent; color:var(--text); border:1px solid rgba(255,255,255,.25); padding:.38rem .6rem; border-radius:.5rem; cursor:pointer }
 .toolbar button[aria-pressed="true"]{ outline:2px solid var(--ring) }
 /* Footer */
 .site-footer{ padding:2rem 1rem 4rem; text-align:center; color:var(--muted) }
 /* A11y: reduce motion */
 @media (prefers-reduced-motion: reduce){ *{ transition:none!important; animation:none!important } }
 /* --- Results grids (for figures) --- */
 .results-group{ margin-top:.8rem }
 .results-group h3{ margin:.2rem 0 .4rem; font-size:1rem; color:var(--muted) }
 .grid-2{
 display:grid; grid-template-columns:repeat(2,1fr); gap:1rem;
 }
 .grid-3{
 display:grid; grid-template-columns:repeat(3,1fr); gap:1rem;
 }
 @media (max-width: 900px){
 .grid-2, .grid-3{ grid-template-columns:1fr }
 }
 .figure-card img{
 width:100%; height:auto; border-radius:.6rem; display:block;
 box-shadow:0 2px 12px rgba(0,0,0,.08);
 }
 .figure-card figcaption{
 font-size:.9rem; color:var(--muted); margin-top:.35rem;
 }
 /* make a card span both columns of .media-grid */
 .full-span { grid-column: 1 / -1; }
 @media (max-width: 920px){ .full-span { grid-column: auto; } }
 /* ===== Image modal (click-to-zoom) ===== */
 .img-modal {
 display:none; position:fixed; inset:0; z-index:1000;
 background:rgba(0,0,0,.75);
 align-items:center; justify-content:center;
 padding:2rem;
 }
 .img-modal.open { display:flex; }
 .img-modal img {
 max-width:92vw; max-height:92vh;
 border-radius:.6rem; box-shadow:0 6px 28px rgba(0,0,0,.4);
 }
 .img-modal .close {
 position:absolute; top:12px; right:16px;
 font-size:1.6rem; color:#fff; background:transparent; border:0; cursor:pointer;
 }
 .figure-card img { cursor: zoom-in; }
 body.modal-open { overflow:hidden; } /* prevent background scroll */
 .btn-row{ display:flex; gap:.6rem; justify-content:center; flex-wrap:wrap; margin:.6rem 0 .2rem; }
 .btn.btn-disabled{ opacity:.6; cursor:not-allowed; pointer-events:none; }
 /* push the hero video down so the button never overlaps */
 .hero > video{ margin-top:.8rem; display:block; max-width:980px; width:100%; border-radius:.8rem; border:var(--border); background:var(--card); }
.hint{
 margin-top:.35rem; font-size:.88rem; color:var(--muted);
 border-left:3px solid color-mix(in oklab, var(--accent), transparent 65%);
 padding:.2rem .6rem; line-height:1.4;
 }
 </style>
</head>
<body>
 <!--
 ASSETS CHECKLIST (drop into your Space):
 - assets/cover.jpg (hero title image)
 - assets/sample_AV.mp4 (example clip – audio+video)
 - assets/sample_A.mp4 (audio-only version)
 - assets/sample_V.mp4 (video-only version)
 - assets/sample_poster.jpg (poster frame for the video)
 - assets/heatmap.png (overall heatmap graphic)
 - assets/confusion.png (confusion matrix or similar)
 - assets/favicon.png (16–64px)
 -->
<header class="site-header">
 <a href="#home" class="logo" aria-label="AVIP Benchmark home"><span class="dot" aria-hidden="true"></span> AVIP<span>Benchmark</span></a>
 <button class="nav-toggle" aria-expanded="false" aria-controls="site-nav">☰</button>
 <nav id="site-nav" class="site-nav" aria-label="Main navigation">
 <a href="#home">Home</a>
 <a href="#about">About</a>
 <a href="#leaderboard">Leaderboard</a>
 <a href="#media">Examples</a>
 <a href="#contact">Contact</a>
 </nav>
 </header>
<main id="home">
 <section class="hero" aria-labelledby="tagline">
 <h1 id="tagline">Do you hear it? Meet AVIP-Bench</h1>
 <p style="font-style:italic;">
 A controlled benchmark for evaluating intuitive physics from video &amp; sound.
 </p>
 <p>
 Objects crash, bounce, and shatter - our benchmark of audiovisual object drops
 probes whether models benefit from adding <strong>sound</strong> when reasoning about physics.
 </p>
 <!-- Button row: examples + PDF -->
 <div class="btn-row">
 <a class="btn" href="#media">See example Videos and Results</a>
 <a id="paperLink" href="#" class="btn btn-disabled" aria-disabled="true">📄 PDF coming soon</a>
 </div>
<!-- Hero video (pushed down a bit) -->
 <video autoplay muted loop playsinline poster="">
 <source src="thumbnail.mp4" type="video/mp4" />
 <!-- Remove the webm line unless you add it later -->
 <source src="thumbnail.webm" type="video/webm" />
 </video>
 </section>
<section id="about" class="section">
 <h2>What is AVIP?</h2>
 <p class="muted">A tiny, controlled benchmark with triplet videos per clip: <span class="pill">A</span> audio-only, <span class="pill">V</span> video-only, and <span class="pill">AV</span> audio+video. Tasks: <em>object</em>, <em>material</em>, <em>outcome</em>. We check top‑1 predictions vs. ground truth and look for cross‑modal gains.</p>
 <ul class="features">
 <li class="card">📦 <strong>Minimal, reproducible clips</strong><br/>Short single‑impact scenes recorded in a controlled setup.</li>
 <li class="card">🔊 <strong>Modality toggles</strong><br/>Each clip exists as A, V, and AV to test true audio usage.</li>
 <li class="card">📈 <strong>Metrics</strong><br/>Top‑1 accuracy per task and an <em>AV − max(A,V)</em> cross‑modal gain.</li>
 <li class="card">🧪 <strong>Probe‑style prompts</strong><br/>Strict label sets &amp; JSON outputs to avoid prompt drift.</li>
 </ul>
 <details class="card" style="margin-top:1rem">
 <summary><strong>Method (short)</strong></summary>
 <ol>
<li>For each clip, run models on A, V, and AV variants with the same instruction-style prompt.</li>
 <li>Decode model outputs into <code>{object, material, outcome}</code> and compare against labels.</li>
 <li>Compute per-task Top-1 and Top-5 accuracy and cross-modal gain per clip and in aggregate; additionally report calibration/confidence metrics (ECE, Brier, margin, entropy, Top-1 probability) and probing-based audio reliance via fixed cue selection and A/V/AV consistency; all metrics computed on the paired clip set (A&cap;V&cap;AV) with 95% confidence intervals.</li>
 </ol>
 </details>
 </section>
<section id="leaderboard" class="section" aria-labelledby="lb-title">
 <h2 id="lb-title">Leaderboard</h2>
 <p class="muted">Per‑Modality (A / V / AV)</p>
 <div class="table-wrap">
 <table aria-describedby="lb-title">
 <thead>
 <tr>
 <th scope="col">Model</th>
 <th scope="col">Modality</th>
 <th scope="col">N</th>
 <th scope="col">Top‑1 Acc (%)</th>
 <th scope="col">Updated</th>
 </tr>
 </thead>
 <tbody id="leaderboard-body"><!-- JS renders here --></tbody>
 </table>
 </div>
<!-- Deine Daten (inline JSON) -->
 <script id="leaderboard-data" type="application/json">{
 "rows": [
 {"model":"Gemini‑2.5 Flash (no think)", "modality":"A", "Top1AccuracyinPercent":20.0, "N":993},
 {"model":"Gemini‑2.5 Flash (no think)", "modality":"AV", "Top1AccuracyinPercent":53.4, "N":993},
 {"model":"Gemini‑2.5 Flash (no think)", "modality":"V", "Top1AccuracyinPercent":48.2, "N":993},
{"model":"Gemini‑2.5 Flash (think)", "modality":"A", "Top1AccuracyinPercent":24.1, "N":990},
 {"model":"Gemini‑2.5 Flash (think)", "modality":"AV", "Top1AccuracyinPercent":58.5, "N":993},
 {"model":"Gemini‑2.5 Flash (think)", "modality":"V", "Top1AccuracyinPercent":50.9, "N":993},
{"model":"Gemini‑2.5 Pro (think)", "modality":"A", "Top1AccuracyinPercent":17.3, "N":819},
 {"model":"Gemini‑2.5 Pro (think)", "modality":"AV", "Top1AccuracyinPercent":61.8, "N":807},
 {"model":"Gemini‑2.5 Pro (think)", "modality":"V", "Top1AccuracyinPercent":56.3, "N":807},
{"model":"Qwen2.5‑Omni 7B (local)", "modality":"A", "Top1AccuracyinPercent":10.9, "N":993},
 {"model":"Qwen2.5‑Omni 7B (local)", "modality":"AV", "Top1AccuracyinPercent":38.7, "N":993},
 {"model":"Qwen2.5‑Omni 7B (local)", "modality":"V", "Top1AccuracyinPercent":38.5, "N":993}
 ]
 }</script>
 </section>
<section id="media" class="section" aria-labelledby="ex-title">
 <h2 id="ex-title">Example clips and Plots</h2>
 <div class="media-grid">
 <figure class="media-card">
 <video id="sampleVideo" controls preload="metadata" playsinline poster="assets/sample_poster.jpg">
 <source src="paperbox_high_1.MP4" type="video/mp4" />
 Your browser doesn’t support HTML5 video.
 </video>
 <div class="toolbar" role="toolbar" aria-label="Version selector">
 <button type="button" class="ver" data-src="paperbox_high_1_A.mp4" aria-pressed="false" aria-label="Audio only (A)">A</button>
 <button type="button" class="ver" data-src="paperbox_high_1_V.mp4" aria-pressed="false" aria-label="Video only (V)">V</button>
 <button type="button" class="ver" data-src="paperbox_high_1.MP4" aria-pressed="true" aria-label="Audio + Video (AV)">AV</button>
 <span class="muted" id="verStatus" aria-live="polite" style="margin-left:.4rem">Now showing: AV</span>
 </div>
 <figcaption class="muted">Task labels (demo): <strong>object</strong>=<code>paperbox</code>, <strong>material</strong>=<code>cardboard</code>, <strong>outcome</strong>=<code>bounce</code></figcaption>
 </figure>
<!-- Group 0: Cross-Modal Gain heatmap -->
 <div class="results-group">
 <h3>Cross-Modal Gain (CMG)</h3>
 <figure class="figure-card">
 <img src="xmod_cis.png" alt="Cross-Modal Gain heatmap" loading="lazy">
 <figcaption>CMG in percentage points per engine; horizontal bars are 95\% paired-bootstrap CIs on the paired clip set.</figcaption>
 <div class="hint">Look for positive values: these mean AV was better than either audio or video alone. Gains usually appear for outcome prediction, but rarely for object or material recognition.</div>
 </figure>
 </div>
<!-- Group 1: Average modality attribution (Audio vs Video) -->
 <div class="results-group">
 <h3>Average modality attribution (AV)</h3>
 <div class="grid-2">
 <figure class="figure-card">
 <img src="Heatmap_Audio.png" alt="Average audio weight across models" loading="lazy">
 <figcaption>Audio weight by model.</figcaption>
 <div class="hint">What to look for: Red = model relies more on audio, Blue = model relies less.
Engines that “listen” more may gain on outcome prediction, but not always.</div>
 </figure>
 <figure class="figure-card">
 <img src="Heatmap_Video.png" alt="Average video weight across models" loading="lazy">
 <figcaption>Video weight by model.</figcaption>
 <div class="hint">What to look for: Red = model relies more on video, Blue = model relies less.
Engines that “look” more often ignore sound, which can explain weak cross-modal gains.</div>
 </figure>
 </div>
 </div>
<!-- Group 2: Top-1 Accuracy: single combined figure -->
 <div class="media-card full-span">
 <div class="results-group">
 <h3>Top-1 accuracy by task</h3>
 <figure class="figure-card">
 <img src="accuracy_micro_macro_cis.png" alt="Top-1 accuracy per model across object, material, and outcome for A, V, AV" loading="lazy">
 <figcaption>Top-1 accuracy with 95% CIs (A, V, AV) across tasks and models.</figcaption>
 <div class="hint">What to look for: V is usually highest; AV improves over A and sometimes nudges past V on outcome.
Big gaps A→AV mean sound is helpful; AV≈V means little extra benefit.</div>
 </figure>
 </div>
 </div>
</section>
<section id="contact" class="section">
 <h2>Contact</h2>
 <p>Questions? <a href="mailto:bramo.g@protonmail.com">bramo.g@protonmail.com</a></p>
 <p class="muted"><a href="https://huggingface.co/Grets/AVIP">huggingface.co/Grets/AVIP</a></p>
 </section>
 </main>
<footer class="site-footer">
 <small>&copy; <span id="year"></span> Grets. Rendered by Hugging Face Spaces.</small>
</footer>
<!-- 1) Basis-Skripte (Nav & Jahr) -->
<script>
 // Mobile nav toggle
 const navToggle = document.querySelector('.nav-toggle');
 const nav = document.getElementById('site-nav');
 if (navToggle && nav) {
 navToggle.addEventListener('click', () => {
 const open = nav.classList.toggle('open');
 navToggle.setAttribute('aria-expanded', String(open));
 });
 }
 // Year in footer
 const y = document.getElementById('year');
 if (y) y.textContent = new Date().getFullYear();
</script>
<!-- 2) Leaderboard-Renderer (nur deine inline-JSON rows) -->
<script>
 (function renderLeaderboard(){
 const el = document.getElementById('leaderboard-body');
 const dataEl = document.getElementById('leaderboard-data');
 if (!el || !dataEl) return;
 // JSON laden
 let rows = [];
 try {
 const parsed = JSON.parse(dataEl.textContent.trim());
 rows = Array.isArray(parsed.rows) ? parsed.rows : [];
 } catch (_) {}
 // Sortierung: Model (A–Z) -> Modality in Reihenfolge AV, V, A
 const order = ['AV', 'V', 'A'];
 rows.sort((a,b) =>
 String(a.model).localeCompare(String(b.model)) ||
 order.indexOf(a.modality) - order.indexOf(b.modality)
 );
 const today = new Date().toISOString().slice(0,10);
 const fmtPct = v => (v==null || v==='') ? '—' : (Number(v).toFixed(1) + '%');
 // Spalten: Model | Modality | N | Top-1 | Top-5 | Updated
 el.innerHTML = rows.map(r => `
 <tr>
 <td>${r.model}</td>
 <td>${r.modality}</td>
 <td>${r.N ?? '—'}</td>
 <td>${fmtPct(r.Top1AccuracyinPercent)}</td>
 <td>${r.updated ?? today}</td>
 </tr>
 `).join('');
 })();
</script>
<!-- 3) Example video toggles -->
<script>
 (function(){
 const video = document.getElementById('sampleVideo');
 const verButtons = document.querySelectorAll('.ver');
 const verStatus = document.getElementById('verStatus');
 verButtons.forEach(btn => {
 btn.addEventListener('click', () => {
 verButtons.forEach(b => b.setAttribute('aria-pressed','false'));
 btn.setAttribute('aria-pressed','true');
 const src = btn.getAttribute('data-src');
 const label = btn.textContent.trim();
 if (src && video) {
 const wasPlaying = !video.paused && !video.ended;
 video.pause();
 video.querySelector('source').src = src;
 video.load();
 if (wasPlaying) video.play().catch(()=>{});
 if (verStatus) verStatus.textContent = `Now showing: ${label}`;
 }
 });
 });
 })();
</script>
<!-- Image modal (click-to-zoom) -->
<div class="img-modal" id="imgModal" aria-hidden="true">
 <button class="close" aria-label="Close">×</button>
 <img id="imgModalImg" alt="">
</div>
<script>
(function(){
 const modal = document.getElementById('imgModal');
 const modalImg = document.getElementById('imgModalImg');
 if (!modal || !modalImg) return;
 document.addEventListener('click', (e)=>{
 const img = e.target.closest('.figure-card img');
 if (!img) return;
 const full = img.getAttribute('data-full');
 modalImg.src = full || img.src;
 modalImg.alt = img.alt || '';
 modal.classList.add('open');
 document.body.classList.add('modal-open');
 modal.setAttribute('aria-hidden','false');
 });
 modal.addEventListener('click', (e)=>{
 if (e.target === modal || e.target.classList.contains('close')) closeModal();
 });
 document.addEventListener('keydown', (e)=>{
 if (e.key === 'Escape' && modal.classList.contains('open')) closeModal();
 });
 function closeModal(){
 modal.classList.remove('open');
 document.body.classList.remove('modal-open');
 modal.setAttribute('aria-hidden','true');
 modalImg.src = '';
 }
})();
</script>
<script>
 // Where your PDF will live
 const PDF_PATH = "AVIP_gbramow_lbreitkopf_iberger.pdf";
 async function enablePdfButton(){
 try{
 const res = await fetch(PDF_PATH, { method:"HEAD", cache:"no-store" });
 if(!res.ok) return; // keep disabled state
 const a = document.getElementById("paperLink");
 if(!a) return;
 a.href = PDF_PATH;
 a.target = "_blank";
 a.rel = "noopener";
 a.textContent = "📄 View PDF";
 a.classList.remove("btn-disabled");
 a.setAttribute("aria-disabled","false");
 }catch(e){ /* keep disabled */ }
 }
 document.addEventListener("DOMContentLoaded", enablePdfButton);
</script>
</body>
</html>