| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1"> |
| <title>JavisDiT - source motion extraction</title> |
| <style> |
| :root{ --bg:#fff; --panel:#f6f7f9; --line:#e2e5ea; --fg:#12151a; --muted:#5c6470; |
| --accent:#3b6ef5; --good:#137a4e; --bad:#a4451c; --chip:#eceff4; --warn:#b3541e; } |
| @media (prefers-color-scheme: dark){ |
| :root{ --bg:#0e1116; --panel:#161b22; --line:#2a313b; --fg:#e6e9ee; --muted:#9aa4b2; |
| --accent:#7aa2ff; --good:#4ad295; --bad:#ff9b6a; --chip:#212832; --warn:#ffb17a; } } |
| :root[data-theme="dark"]{ --bg:#0e1116; --panel:#161b22; --line:#2a313b; --fg:#e6e9ee; |
| --muted:#9aa4b2; --accent:#7aa2ff; --good:#4ad295; --bad:#ff9b6a; --chip:#212832; --warn:#ffb17a; } |
| :root[data-theme="light"]{ --bg:#fff; --panel:#f6f7f9; --line:#e2e5ea; --fg:#12151a; |
| --muted:#5c6470; --accent:#3b6ef5; --good:#137a4e; --bad:#a4451c; --chip:#eceff4; --warn:#b3541e; } |
| *{box-sizing:border-box} |
| body{margin:0;background:var(--bg);color:var(--fg); |
| font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif} |
| .wrap{max-width:1240px;margin:0 auto;padding:32px 20px 80px} |
| h1{font-size:23px;margin:0 0 6px;letter-spacing:-.01em} |
| h2{font-size:17px;margin:34px 0 6px} |
| p{margin:8px 0} .sub{color:var(--muted);margin:0 0 14px} |
| code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.9em; |
| background:var(--chip);padding:1px 5px;border-radius:4px} |
| .chips{display:flex;flex-wrap:wrap;gap:8px;margin:14px 0} |
| .chip{background:var(--chip);border:1px solid var(--line);border-radius:999px; |
| padding:3px 11px;font-size:12.5px;color:var(--muted)} |
| .chip b{color:var(--fg);font-weight:600} |
| .note{background:var(--panel);border:1px solid var(--line);border-left:3px solid var(--warn); |
| border-radius:8px;padding:12px 15px;margin:18px 0} |
| .note.ok{border-left-color:var(--good)} |
| .note ul{margin:6px 0 0;padding-left:20px} .note li{margin:4px 0} |
| table{border-collapse:collapse;width:100%;font-size:13px;margin-top:10px;display:block; |
| overflow-x:auto;white-space:nowrap} |
| th,td{border-bottom:1px solid var(--line);padding:7px 10px;text-align:right} |
| th:first-child,td:first-child{text-align:left} |
| th{color:var(--muted);font-weight:600;font-size:11.5px;text-transform:uppercase} |
| td.g{color:var(--good);font-weight:600} td.b{color:var(--bad);font-weight:600} |
| .bar{position:sticky;top:0;background:var(--bg);z-index:5;padding:10px 0 0; |
| border-bottom:1px solid var(--line);margin:22px 0 8px} |
| .picker{display:flex;flex-wrap:wrap;gap:8px;padding-bottom:10px} |
| .picker button{font:inherit;font-size:13.5px;cursor:pointer;border:1px solid var(--line); |
| background:var(--panel);color:var(--fg);border-radius:8px;padding:6px 13px;text-align:left} |
| .picker button.on{background:var(--accent);border-color:var(--accent);color:#fff;font-weight:600} |
| .picker button small{display:block;opacity:.75;font-size:11px} |
| .card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px;margin:14px 0} |
| .wide video{width:100%;border-radius:8px;background:#000;display:block;border:1px solid var(--line)} |
| .lgd{display:flex;flex-wrap:wrap;gap:14px;margin-top:9px;font-size:11.5px;color:var(--muted)} |
| .lgd b{color:var(--fg);font-weight:600} |
| .foot{color:var(--muted);font-size:12.5px;margin-top:40px;border-top:1px solid var(--line);padding-top:14px} |
| </style> |
| </head> |
| <body> |
| <div class="wrap"> |
| <h1>Source motion, extracted for the edit finetune</h1> |
| <p class="sub">Ten held-out test clips. Each source video is turned into a dense motion |
| field on the video-latent grid, so it can be channel-concatenated to the noisy latent |
| as extra conditioning — and, separately, reduced back to the single line a person |
| would have drawn to produce it.</p> |
| <div class="chips" id="chips"></div> |
|
|
| <p class="sub"><code>video → AllTracker reference flow (81 frames @ 256×256) |
| → forward splat into frame-t coordinates → causal pool to 21×32×32 |
| → [3, 21, 32, 32] fp16 (dx, dy, validity)</code></p> |
|
|
| <div class="note"> |
| <b>Two things this measurement says, before any training.</b> |
| <ul> |
| <li><b>The temporal pooling is lossy on this data.</b> Wan's VAE is causal, so each |
| latent frame covers four pixel frames and the flow must be reduced over that |
| group. The within-group spread — how much the displacement changes inside |
| one group — has a median of <b id="cspread">—</b> px here, against a |
| latent cell of 8 px. The mean is blending motions several cells apart, which |
| argues for <code>--temporal_reduce last</code> on this corpus.</li> |
| <li><b>Validity is not a formality.</b> Median coverage is |
| <b id="cvalid">—</b>: on fast clips a third to a half of the frame is |
| content that did not exist in frame 0 (disocclusion, or influx from outside), |
| where the field carries no information at all. That is what the third channel is |
| for, and why <code>validity == 0 ⇒ dx = dy = 0</code> is enforced everywhere |
| — so an all-zero tensor reads as "no information", not "the scene is |
| static".</li> |
| </ul> |
| </div> |
|
|
| <h2>Per clip</h2> |
| <table id="tbl"></table> |
|
|
| <div class="note ok"> |
| <b>Only <span id="ndraw">—</span> of 10 clips are expressible as one drawn line.</b> |
| <code>wander</code> is path length over net displacement. A hand-drawn drag is a single |
| monotone stroke, so a region that oscillates — a strumming arm, water, a bird |
| flapping — has a long path and almost no net travel, and no single line can |
| represent it. The first version of this tool happily emitted a zig-zag for those; it now |
| says so instead. Clips with <code>wander < 2</code> and real travel are the ones where |
| a drag is actually the right control surface. |
| </div> |
|
|
| <div class="bar"><div class="picker" id="picker"></div></div> |
| <div id="cards"></div> |
|
|
| <p class="foot">AllTracker (window 16, 4 iters) on the exact 81 frames the cached VAE |
| latents were built from, via JavisDiT's own <code>load_video_at_target_fps</code>. |
| The splat is a pure-PyTorch reimplementation of softsplat's <code>avg</code> mode |
| (cupy is unavailable here), validated against closed-form answers: exact on integer and |
| sub-pixel translation, matching the inverse area Jacobian 1/s² to 0.3% on an affine |
| zoom, and exactly zero round-trip error on locally-constant flow. Latent 0 is forced to |
| exactly (0, 0, 1) — AllTracker leaves a ~0.2–0.4 px self-regression residual |
| at frame 0 that would otherwise manufacture spurious sub-pixel holes.</p> |
| </div> |
|
|
| <script> |
| let D=null; |
| const f2=(v,d)=>(v===null||v===undefined||isNaN(v))?"—":v.toFixed(d); |
| const med=(a)=>{const s=[...a].sort((x,y)=>x-y);return s[Math.floor(s.length/2)];}; |
| function chips(){ |
| const sp=med(D.clips.map(r=>r.within_group_spread_px)); |
| const va=med(D.clips.map(r=>r.valid_frac)); |
| const nd=D.clips.filter(r=>r.drawable).length; |
| document.getElementById("cspread").textContent=f2(sp,1); |
| document.getElementById("cvalid").textContent=f2(va,3); |
| document.getElementById("ndraw").textContent=nd; |
| const c=[["clips",D.clips.length],["tracker",D.config.tracker], |
| ["field","[3, 21, 32, 32]"], |
| ["median |flow|",f2(med(D.clips.map(r=>r.mean_disp_px)),1)+" px"], |
| ["drawable",nd+" / "+D.clips.length]]; |
| document.getElementById("chips").innerHTML=c.map(([k,v])=>`<span class="chip">${k} <b>${v}</b></span>`).join(""); |
| } |
| function table(){ |
| const rows=D.clips.map((r,i)=>`<tr><td><a href="#" onclick="show(${i});return false">${r.video_id}</a></td> |
| <td>${f2(r.mean_disp_px,1)}</td><td>${f2(r.p95_disp_px,1)}</td> |
| <td class="${r.valid_frac<0.5?'b':'g'}">${f2(r.valid_frac,3)}</td> |
| <td class="${r.within_group_spread_px>8?'b':'g'}">${f2(r.within_group_spread_px,1)}</td> |
| <td>${f2(r.t0_residual_px,3)}</td><td>${r.n_clicks}</td> |
| <td>${f2(r.net_travel_px,1)}</td> |
| <td class="${r.wander<2?'g':'b'}">${f2(r.wander,2)}</td> |
| <td>${r.drawable?"yes":"no"}</td></tr>`).join(""); |
| document.getElementById("tbl").innerHTML= |
| `<thead><tr><th>clip</th><th>mean |flow| px</th><th>p95 px</th><th>validity</th> |
| <th>within-group spread px</th><th>t0 residual px</th><th>clicks</th> |
| <th>net travel px</th><th>wander</th><th>drawable</th></tr></thead><tbody>${rows}</tbody>`; |
| } |
| function picker(){ |
| document.getElementById("picker").innerHTML=D.clips.map((r,i)=> |
| `<button id="pk${i}" onclick="show(${i})">${r.video_id}<small>${r.drawable?"drawable":"oscillates"} · ${f2(r.mean_disp_px,1)} px</small></button>`).join(""); |
| } |
| function show(i){ |
| D.clips.forEach((_,k)=>{const b=document.getElementById("pk"+k); if(b) b.className=(k===i)?"on":"";}); |
| const r=D.clips[i]; |
| document.getElementById("cards").innerHTML=`<div class="card"> |
| <div style="display:flex;gap:10px;align-items:baseline;flex-wrap:wrap"><b>${r.video_id}</b> |
| <code>${r.drawable?"drawable":"oscillates"}</code></div> |
| <div class="wide"><video src="videos/${r.clip}" controls loop muted playsinline preload="metadata"></video></div> |
| <div class="lgd"> |
| <span><b>input</b> the 21 causal group-end frames</span> |
| <span><b>dense motion</b> hue = direction, brightness = speed; black at t=0 by construction</span> |
| <span><b>validity</b> white = the tracker saw this cell, black = disoccluded</span> |
| <span><b>one drawable line</b> grey = the dense path, red = the ${r.n_clicks} clicks it reduces to</span> |
| </div> |
| <div class="lgd"><span>mean |flow| <b>${f2(r.mean_disp_px,1)}</b> px</span> |
| <span>validity <b>${f2(r.valid_frac,3)}</b></span> |
| <span>within-group spread <b>${f2(r.within_group_spread_px,1)}</b> px</span> |
| <span>net travel <b>${f2(r.net_travel_px,1)}</b> px over a path of <b>${f2(r.path_len_px,1)}</b></span> |
| <span>wander <b>${f2(r.wander,2)}</b></span></div> |
| <div class="lgd"><span>clicks: <code>${JSON.stringify(r.clicks)}</code></span></div></div>`; |
| } |
| fetch("data.json").then(r=>r.json()).then(d=>{D=d;chips();table();picker();show(0);}); |
| </script> |
| </body> |
| </html> |
|
|
