| { |
| "order": [ |
| "rigid3", |
| "gel3", |
| "gel5" |
| ], |
| "pretty": { |
| "rigid3": "rigid frame (control)", |
| "gel3": "gel frame", |
| "gel5": "gel frame + force" |
| }, |
| "short": { |
| "conventions": { |
| "static (no warp)": 37.122818006243875, |
| "+F (as stored)": 39.10664607989698, |
| "-F (sign flipped)": 36.0492278043432, |
| "+F, xy swapped": 36.78770993917315, |
| "+dx, -dy": 37.26519215760291, |
| "-dx, +dy": 37.14506638373715 |
| }, |
| "stats": { |
| "rigid3": { |
| "epe": 0.8494208560706201, |
| "cos": 0.8057822177334774, |
| "mag_pred": 1.0257021683183583, |
| "mag_gt": 1.3960966996171258, |
| "mag_ratio": 0.7346927820971522, |
| "n": 352 |
| }, |
| "gel3": { |
| "epe": 0.8425940814936026, |
| "cos": 0.8076414258473299, |
| "mag_pred": 1.1331827567720956, |
| "mag_gt": 1.3960966996171258, |
| "mag_ratio": 0.811679274854576, |
| "n": 352 |
| }, |
| "gel5": { |
| "epe": 0.8539616338358346, |
| "cos": 0.8070749697402458, |
| "mag_pred": 1.0990949798883363, |
| "mag_gt": 1.3960966996171258, |
| "mag_ratio": 0.787262787878346, |
| "n": 352 |
| } |
| }, |
| "note": "100 held-out adjacent pairs. The stored flow, warped forward as-is, wins by ~2 dB and every alternative loses — so sign, axis order and scale are right. A global gain sweep peaks at 1.1 and the y-scale at 0.9, both essentially 1, confirming the magnitude calibration too.", |
| "caption": "Held-out samples, one transition (0.83 s) per row plus the +2 columns. 'warp by GT flow' is the ceiling this warp operator allows. Note that 'warp by pred' sometimes BEATS it -- under-predicted flow warps less, and warping hurts these images, so PSNR rewards being timid. Judge the flow from the quivers and EPE, not from the warped-frame PSNR." |
| }, |
| "pixel": { |
| "rows": [ |
| { |
| "name": "raw pixel, per frame-pair", |
| "grid": "24×32", |
| "unit": "1 pair", |
| "cos": 0.461, |
| "ratio": 0.259, |
| "best": false |
| }, |
| { |
| "name": "raw pixel, 5-pair sum", |
| "grid": "24×32", |
| "unit": "5 pairs", |
| "cos": 0.645, |
| "ratio": 0.437, |
| "best": false |
| }, |
| { |
| "name": "raw pixel, 5-pair sum", |
| "grid": "16×16", |
| "unit": "5 pairs", |
| "cos": 0.664, |
| "ratio": 0.455, |
| "best": false |
| }, |
| { |
| "name": "VAE latent (existing)", |
| "grid": "16×16", |
| "unit": "5 pairs", |
| "cos": 0.807, |
| "ratio": 0.787, |
| "best": true |
| } |
| ], |
| "abl": [ |
| [ |
| "action only", |
| 1.296, |
| 0.534 |
| ], |
| [ |
| "state only", |
| 1.291, |
| 0.526 |
| ], |
| [ |
| "both", |
| 1.183, |
| 0.645 |
| ] |
| ], |
| "note": "<strong>Pixel space is worse here, not better — the opposite of what I expected.</strong> Two lessons. First, my initial run changed the state <em>and</em> the temporal unit at once; going to single frame-pairs put the target at median 0.16 px, sub-pixel and at AllTracker's noise floor. Restoring the 5-pair sum lifted cos 0.46 → 0.645, so the old chunk-summing was buying ~√5 of SNR, not just bookkeeping. Second, even matched on grid and temporal unit the latent still wins. The 45% encode variance is a property of the latent <em>change</em>; as a static input the 48-channel pretrained latent still beats a 96×128 RGB frame through a 2-layer stem trained from scratch on 74k samples. The gap looks like the state encoder, not the grid or the units." |
| }, |
| "view": { |
| "rows": [ |
| { |
| "name": "ActionImage + camera frame (pixview_D)", |
| "epe": 2.5494974680849944, |
| "cos": 0.8121550327224547, |
| "ratio": 0.8850180916220904, |
| "best": true |
| }, |
| { |
| "name": "ActionImage only", |
| "epe": 3.057011372772778, |
| "cos": 0.7729672869418572, |
| "ratio": 0.8596414098603565, |
| "best": false |
| }, |
| { |
| "name": "camera frame only", |
| "epe": 5.124187510738971, |
| "cos": 0.42765343934664707, |
| "ratio": 0.42760698851879564, |
| "best": false |
| }, |
| { |
| "name": "vmx_E (VAE latent, 16×16, px@256crop)", |
| "epe": null, |
| "cos": 0.8653, |
| "ratio": null, |
| "best": false |
| } |
| ], |
| "note": "Pixel space costs far less here than on the tactile branch: cos 0.812 vs vmx_E's 0.8653, a gap of 0.05, where the tactile pixel stage lost by 0.14. And magnitude ratio <strong>0.885</strong> is the best measured anywhere in this study — better than the tactile latent encoder (0.787) and better than an explicit magnitude penalty reached (0.862). Caveats: EPE is not comparable across grids/units (only cos and ratio are), a 24×32 grid is intrinsically harder than 16×16, vmx_E also had the analytic physics prior enabled which this does not, and this is a single seed.", |
| "warp": [ |
| { |
| "name": "nothing (copy frame t)", |
| "psnr": 21.458693626944864, |
| "d": 0, |
| "best": false |
| }, |
| { |
| "name": "the PREDICTED flow", |
| "psnr": 23.821329628384042, |
| "d": 2.362636001439178, |
| "best": true |
| }, |
| { |
| "name": "the ground-truth flow (ceiling)", |
| "psnr": 25.50534211064509, |
| "d": 4.046648483700224, |
| "best": false |
| } |
| ], |
| "warpnote": "310 held-out transitions. The prediction recovers <strong>58%</strong> of what a perfect flow buys, and a perfect flow buys 4.05 dB — a real gain, unlike the tactile branch where warping lost to doing nothing." |
| }, |
| "inventory": [ |
| { |
| "run": "amx_A", |
| "epochs": 6, |
| "epe_act": 1.3913597703061666, |
| "cos": 0.5522186694061856, |
| "ratio": null |
| }, |
| { |
| "run": "amx_B", |
| "epochs": 6, |
| "epe_act": 1.3620053482609142, |
| "cos": 0.5532014349035909, |
| "ratio": null |
| }, |
| { |
| "run": "amx_C", |
| "epochs": 6, |
| "epe_act": 1.1900236889568292, |
| "cos": 0.6765851756644143, |
| "ratio": null |
| }, |
| { |
| "run": "amx_D", |
| "epochs": 6, |
| "epe_act": 1.14568726080726, |
| "cos": 0.7209199503980377, |
| "ratio": null |
| }, |
| { |
| "run": "amx_E", |
| "epochs": 6, |
| "epe_act": 1.135178759677794, |
| "cos": 0.720063670145471, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel3_s0", |
| "epochs": 12, |
| "epe_act": 1.1345561851465333, |
| "cos": 0.7271676416359532, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel3_s1", |
| "epochs": 12, |
| "epe_act": 1.1431138532674148, |
| "cos": 0.7233914384978462, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel3_s2", |
| "epochs": 12, |
| "epe_act": 1.1438794515031474, |
| "cos": 0.7213954719641202, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5_s0", |
| "epochs": 12, |
| "epe_act": 1.1403946355119585, |
| "cos": 0.729475603888566, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5_s1", |
| "epochs": 12, |
| "epe_act": 1.135017718254689, |
| "cos": 0.7303181343427246, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5_s2", |
| "epochs": 12, |
| "epe_act": 1.1286770551586096, |
| "cos": 0.7302202065368937, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5sum_s0", |
| "epochs": 12, |
| "epe_act": 1.142919810178045, |
| "cos": 0.7311607766013426, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5sum_s1", |
| "epochs": 12, |
| "epe_act": 1.125809685744154, |
| "cos": 0.7344135940393155, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel5sum_s2", |
| "epochs": 12, |
| "epe_act": 1.124449494193186, |
| "cos": 0.7333485318595235, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel_D_noforce", |
| "epochs": 12, |
| "epe_act": 1.1345561851465333, |
| "cos": 0.7271676416359532, |
| "ratio": null |
| }, |
| { |
| "run": "amx_gel_D_sumbug", |
| "epochs": 12, |
| "epe_act": 1.142919810178045, |
| "cos": 0.7311607766013426, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag0p0_s0", |
| "epochs": 12, |
| "epe_act": 1.1403946355119585, |
| "cos": 0.729475603888566, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag0p25_s0", |
| "epochs": 12, |
| "epe_act": 1.143256354501048, |
| "cos": 0.7279246137276336, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag1p0_s0", |
| "epochs": 12, |
| "epe_act": 1.1511297105282785, |
| "cos": 0.7248073978981037, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag2p0_s0", |
| "epochs": 12, |
| "epe_act": 1.1498703407985686, |
| "cos": 0.7247010417561103, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag4p0_s0", |
| "epochs": 12, |
| "epe_act": 1.1587979376239825, |
| "cos": 0.7205953078747355, |
| "ratio": null |
| }, |
| { |
| "run": "amx_mag8p0_s0", |
| "epochs": 12, |
| "epe_act": 1.1708741522813888, |
| "cos": 0.6997621089217619, |
| "ratio": null |
| }, |
| { |
| "run": "amx_overfit_test", |
| "epochs": 3, |
| "epe_act": 1.4168758958117293, |
| "cos": 0.560820230990249, |
| "ratio": null |
| }, |
| { |
| "run": "amx_rigid3_s0", |
| "epochs": 12, |
| "epe_act": 1.1242302951004741, |
| "cos": 0.7254833644338937, |
| "ratio": null |
| }, |
| { |
| "run": "amx_rigid3_s1", |
| "epochs": 12, |
| "epe_act": 1.14148026541209, |
| "cos": 0.7111668539477364, |
| "ratio": null |
| }, |
| { |
| "run": "amx_rigid3_s2", |
| "epochs": 12, |
| "epe_act": 1.1461318684403463, |
| "cos": 0.7207701281224062, |
| "ratio": null |
| }, |
| { |
| "run": "pix16_D", |
| "epochs": 8, |
| "epe_act": 1.1665619398277647, |
| "cos": 0.664278115245952, |
| "ratio": 0.45502508996721563 |
| }, |
| { |
| "run": "pix_D", |
| "epochs": 8, |
| "epe_act": 0.3921590565324571, |
| "cos": 0.46217542797179006, |
| "ratio": 0.26423094648797807 |
| }, |
| { |
| "run": "pixc5_B", |
| "epochs": 8, |
| "epe_act": 1.2962111514125592, |
| "cos": 0.5384872265277628, |
| "ratio": 0.3522059132106803 |
| }, |
| { |
| "run": "pixc5_C", |
| "epochs": 8, |
| "epe_act": 1.2912984236771061, |
| "cos": 0.526156911269827, |
| "ratio": 0.3206272200948541 |
| }, |
| { |
| "run": "pixc5_D", |
| "epochs": 8, |
| "epe_act": 1.182782718315735, |
| "cos": 0.6452763895117689, |
| "ratio": 0.43740873307592 |
| } |
| ], |
| "inventory_note": "31 runs on this held-out split. amx_A–amx_E and amx_overfit_test predate this work; everything else was trained here. Magnitude ratio is only logged by the pixel trainer, so it is blank for the amx runs — those numbers appear in section 4 instead. EPE is not comparable across the amx (px@256, 16×16) and pix (native px) families; cos is.", |
| "threshold": { |
| "bins": [ |
| [ |
| 0.05253026261925697, |
| 0.07807341124862432, |
| 117, |
| -0.19727820782156286 |
| ], |
| [ |
| 0.07807341124862432, |
| 0.09907817095518112, |
| 117, |
| -0.1960828857793335 |
| ], |
| [ |
| 0.09907817095518112, |
| 0.14265959151089191, |
| 117, |
| -0.13654598260528006 |
| ], |
| [ |
| 0.14265959151089191, |
| 0.22055771201848984, |
| 117, |
| -0.04022311251574792 |
| ], |
| [ |
| 0.22055771201848984, |
| 0.28813134133815765, |
| 117, |
| -0.001130071878249868 |
| ], |
| [ |
| 0.28813134133815765, |
| 0.3947170004248619, |
| 117, |
| 0.026607671375466314 |
| ], |
| [ |
| 0.3947170004248619, |
| 0.5892158672213554, |
| 117, |
| 0.05907286318920096 |
| ], |
| [ |
| 0.5892158672213554, |
| 2.3149893283843994, |
| 117, |
| 0.11565813379280289 |
| ] |
| ], |
| "note": "936 held-out adjacent pairs. Warping only starts paying above about <strong>0.29 px</strong> of mean motion, and <strong>62% of pairs fall below that</strong> — for most adjacent frames, resampling blur costs more than the sub-pixel shift is worth. Note this is the 0.167 s adjacent step; a latent transition spans 5 of them and its flow is ~5× larger, so most transitions clear the threshold. Still worth a magnitude gate before warping anything." |
| }, |
| "bc": { |
| "order": [ |
| "static", |
| "alltracker", |
| "farneback", |
| "oracle" |
| ], |
| "label": { |
| "static": "static (no warp)", |
| "alltracker": "AllTracker ground-truth flow (a tracker)", |
| "farneback": "Farneback (optimises brightness constancy)", |
| "oracle": "oracle: per-pixel best match, ±5 px" |
| }, |
| "psnr": { |
| "static": 35.942847308705176, |
| "alltracker": 38.19904860097141, |
| "farneback": 37.39655808067961, |
| "oracle": 41.640558124103045 |
| }, |
| "note": "40 held-out adjacent pairs. <strong>AllTracker beats Farneback</strong> — a method whose whole objective is to make warping reproduce the next frame reconstructs it <em>worse</em> than the tracker. So the flow is not the weak link. The oracle gains more, but only by using displacements that disagree with the true motion by 2.48 px — larger than the motion itself (1.68 px). It is fitting shading and noise, not recovering motion." |
| }, |
| "why": { |
| "order": [ |
| "darea", |
| "div", |
| "dforce" |
| ], |
| "label": { |
| "dforce": "out-of-plane press (|Δforce|)", |
| "darea": "contact created / destroyed (|Δarea|)", |
| "div": "in-plane compression (|div F|)" |
| }, |
| "rows": { |
| "darea": { |
| "raw": 0.5872729777930227, |
| "flow": 0.40604139250551624, |
| "partial": 0.4679464083671336, |
| "span": "0.637 → 0.944" |
| }, |
| "div": { |
| "raw": 0.6955365954805716, |
| "flow": 0.8449497151431501, |
| "partial": 0.1718241212159736, |
| "span": "0.592 → 1.046" |
| }, |
| "dforce": { |
| "raw": 0.2038955946036579, |
| "flow": 0.2280646771711891, |
| "partial": 0.05062961586935842, |
| "span": "0.769 → 0.869" |
| } |
| }, |
| "note": "626 held-out pairs; mean residual after a ground-truth warp 0.803/255 against a noise floor of 0.297. <strong>Contact creation and destruction dominates</strong> (partial 0.47, residual +48% at fixed motion). Compression is secondary and still partly confounded. Out-of-plane press is <em>weak</em> on its own (partial 0.05) — a plausible-sounding mechanism the data does not support once motion is controlled." |
| }, |
| "magfix": { |
| "order": [ |
| "0.0", |
| "0.25", |
| "1.0", |
| "2.0", |
| "4.0", |
| "8.0" |
| ], |
| "stats": { |
| "0.0": { |
| "epe": 0.8539617552675984, |
| "cos": 0.8070748819630932, |
| "mag_pred": 1.0990955146368255, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.7872631709094708, |
| "n": 352 |
| }, |
| "0.25": { |
| "epe": 0.8597460983913731, |
| "cos": 0.8093682973783209, |
| "mag_pred": 1.1410055116496303, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.8172825793245888, |
| "n": 352 |
| }, |
| "1.0": { |
| "epe": 0.8636800429826095, |
| "cos": 0.8035165485859828, |
| "mag_pred": 1.2031704112887383, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.861810225336614, |
| "n": 352 |
| }, |
| "2.0": { |
| "epe": 0.866651744983921, |
| "cos": 0.8046613101393053, |
| "mag_pred": 1.2020067255944014, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.8609766973333919, |
| "n": 352 |
| }, |
| "4.0": { |
| "epe": 0.8691599562721835, |
| "cos": 0.7967279284546914, |
| "mag_pred": 1.2363024086437442, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.8855421039121397, |
| "n": 352 |
| }, |
| "8.0": { |
| "epe": 0.874935270011933, |
| "cos": 0.780608971211636, |
| "mag_pred": 1.2021466287022287, |
| "mag_gt": 1.3960966996171258, |
| "ratio": 0.8610769075178767, |
| "n": 352 |
| } |
| }, |
| "note": "Seed 0, 12 epochs, held-out. EPE rising is the mechanism confirming itself — EPE is minimised by shrinkage, so any anti-shrinkage term must cost it. But read the per-cell panel before trusting the aggregate: the ratio gain comes largely from <em>over</em>-predicting small flows, while the large-flow deficit (~0.63 → 0.66) is barely touched and the per-cell slope moves only 0.62 → 0.65. Single seed, and the EPE spread across seeds is ~0.01, so treat the EPE column as indicative." |
| }, |
| "byevent": { |
| "order": [ |
| "idle", |
| "onset", |
| "sustained", |
| "release" |
| ], |
| "label": { |
| "idle": "idle (no contact → no contact)", |
| "onset": "onset (no contact → contact)", |
| "sustained": "sustained (contact → contact)", |
| "release": "release (contact → no contact)" |
| }, |
| "bins": { |
| "idle": { |
| "n": 169, |
| "gt_mag": 0.9408222472173928, |
| "d_img": 0.895201954615892, |
| "rigid3": { |
| "epe": 0.4243584687452345, |
| "cos": 0.8898629210301523, |
| "ratio": 0.8376926919647457 |
| }, |
| "gel3": { |
| "epe": 0.41578161180078166, |
| "cos": 0.89516770194562, |
| "ratio": 0.8444037452722207 |
| }, |
| "gel5": { |
| "epe": 0.4171516524381657, |
| "cos": 0.894703471471043, |
| "ratio": 0.8271769212900534 |
| } |
| }, |
| "onset": { |
| "n": 48, |
| "gt_mag": 1.6890246036152046, |
| "d_img": 1.8497557416558266, |
| "rigid3": { |
| "epe": 1.0984720401465893, |
| "cos": 0.7785308607336546, |
| "ratio": 0.7204477048168579 |
| }, |
| "gel3": { |
| "epe": 1.0681375455525186, |
| "cos": 0.791505794144339, |
| "ratio": 0.7463941860737071 |
| }, |
| "gel5": { |
| "epe": 1.0734005394495196, |
| "cos": 0.7891267101383872, |
| "ratio": 0.7419883261124293 |
| } |
| }, |
| "sustained": { |
| "n": 86, |
| "gt_mag": 2.1369912811489993, |
| "d_img": 2.1791713875393537, |
| "rigid3": { |
| "epe": 1.5276011643252636, |
| "cos": 0.6837468345382416, |
| "ratio": 0.7564489261586538 |
| }, |
| "gel3": { |
| "epe": 1.5175656284703762, |
| "cos": 0.6921749928525773, |
| "ratio": 0.7987338427540867 |
| }, |
| "gel5": { |
| "epe": 1.5246289612710937, |
| "cos": 0.6921420429161814, |
| "ratio": 0.7931209508069726 |
| } |
| }, |
| "release": { |
| "n": 49, |
| "gt_mag": 1.3790336190437784, |
| "d_img": 1.7515593694180858, |
| "rigid3": { |
| "epe": 0.9537766946821798, |
| "cos": 0.7210345288016357, |
| "ratio": 0.8075540313104383 |
| }, |
| "gel3": { |
| "epe": 0.9715966369424548, |
| "cos": 0.7233784541386324, |
| "ratio": 0.8427694697363847 |
| }, |
| "gel5": { |
| "epe": 0.9690332834412451, |
| "cos": 0.7377250319232747, |
| "ratio": 0.8655338524555675 |
| } |
| } |
| }, |
| "note": "352 held-out transitions, 3 seeds, identical windows across variants. <strong>169 of them (48%) are idle</strong>, where all three variants tie — which is most of why the aggregate held-out number came out a wash. The gel variants' magnitude advantage lives in the contact regimes, and grows with how much the video changes." |
| }, |
| "explains": { |
| "kv": [ |
| [ |
| "mean ground-truth motion", |
| "1.14 px per 0.167 s step" |
| ], |
| [ |
| "mean |I(t+1) − I(t)|", |
| "1.79 / 255" |
| ], |
| [ |
| "noise + illumination floor", |
| "0.31 / 255 (quietest pairs, no motion at all)" |
| ], |
| [ |
| "motion-attributable change", |
| "1.48 / 255" |
| ], |
| [ |
| "resolution cost, 16×16 vs dense", |
| "0.12 dB — negligible" |
| ], |
| [ |
| "operator cost, backward vs splat", |
| "0.07 dB — negligible" |
| ], |
| [ |
| "<strong>explained by a perfect dense flow</strong>", |
| "<strong>13%</strong> of the total change, 15% of the motion-attributable part" |
| ], |
| [ |
| "|dI| in moving vs still regions", |
| "2.55 vs 0.83 / 255 (corr with |flow| +0.41)" |
| ] |
| ], |
| "caption": "Neither the 16x16 grid nor the warp operator is the bottleneck -- they cost 0.12 and 0.07 dB. A perfect, dense, correctly-splatted flow still gains only ~2.2 dB over doing nothing." |
| }, |
| "longterm": { |
| "drift": { |
| "test": { |
| "rigid3": [ |
| 1.3449834395884535, |
| 1.7661714237669808, |
| 2.263221908043358, |
| 2.620980885441051, |
| 2.6328423616248235, |
| 2.746752045346632, |
| 2.8577525228480467, |
| 3.08537275320719, |
| 3.323568522888905, |
| 3.3343401796111665, |
| 3.452599630303023, |
| 3.641918295332803, |
| 3.779626951717249, |
| 3.8968673599005075, |
| 4.044586537158344, |
| 4.137865290556217, |
| 4.769688435817955, |
| 4.838321066999286, |
| 5.377463296212574, |
| 5.567867063917172 |
| ], |
| "gel3": [ |
| 1.3353903147676522, |
| 1.821856510783559, |
| 2.3685225289806766, |
| 2.763248187163846, |
| 2.7898539982704236, |
| 2.91952167364101, |
| 2.966718479893346, |
| 3.1943293727865067, |
| 3.4468246384743773, |
| 3.448667388947703, |
| 3.605850571379712, |
| 3.861361897222235, |
| 3.99447475950874, |
| 4.141446395573396, |
| 4.249324658563752, |
| 4.2641786279721305, |
| 4.871395982226127, |
| 4.974901353186785, |
| 5.470477952026337, |
| 5.692797018563497 |
| ], |
| "gel5": [ |
| 1.3633004730954343, |
| 1.838484304667771, |
| 2.343277270581108, |
| 2.772037480338113, |
| 2.8386500739293874, |
| 2.951148248712796, |
| 3.0636873370543407, |
| 3.323992843019789, |
| 3.545932750268661, |
| 3.559628958032643, |
| 3.7262839697576493, |
| 3.9406793614887317, |
| 4.062300877480208, |
| 4.1513155519263965, |
| 4.233195268308143, |
| 4.416091782872696, |
| 5.065137285169311, |
| 5.171754041320555, |
| 5.645810339690413, |
| 5.85763385483351 |
| ], |
| "zero": [ |
| 1.9060299855941327, |
| 2.719808954991419, |
| 3.088302637129618, |
| 3.542594632754737, |
| 3.849147020257373, |
| 4.3713874083628825, |
| 4.733975646300624, |
| 5.417159363736925, |
| 5.9213335148532815, |
| 6.174291575672702, |
| 6.388263567861676, |
| 6.894445488336092, |
| 7.304299007470796, |
| 7.831166874100991, |
| 8.530863680147084, |
| 9.136823585619188, |
| 9.630429340959848, |
| 9.840542417735046, |
| 11.034902508144423, |
| 11.471459388136058 |
| ] |
| }, |
| "train": { |
| "rigid3": [ |
| 1.5993235156815215, |
| 2.117592031573979, |
| 2.816777963279685, |
| 3.214726370456369, |
| 3.679569018975847, |
| 4.057619794730067, |
| 4.333859267362663, |
| 4.62309778706804, |
| 4.848084870427532, |
| 5.25019408667772, |
| 5.537762714539304, |
| 5.965742357871588, |
| 6.097840569363486, |
| 6.237047060398754, |
| 6.730128180365446, |
| 6.857237045085087, |
| 7.37162314324697, |
| 7.817763005605595, |
| 8.20789233553764, |
| 8.476939864346742 |
| ], |
| "gel3": [ |
| 1.5825108761971602, |
| 2.056081331940447, |
| 2.696628526189362, |
| 3.094922729428564, |
| 3.508039344574273, |
| 3.879547873043708, |
| 4.125191056410673, |
| 4.427792607563209, |
| 4.636417630683769, |
| 4.983787115052455, |
| 5.2603725789453675, |
| 5.633681249252656, |
| 5.740829391378996, |
| 5.872620719370114, |
| 6.315744293645281, |
| 6.433371254734899, |
| 6.895309123190641, |
| 7.361035929136143, |
| 7.69340694504102, |
| 8.008170351975794 |
| ], |
| "gel5": [ |
| 1.5314663011790053, |
| 1.9783173925617734, |
| 2.64601897091692, |
| 3.0076441706115564, |
| 3.4547836733585155, |
| 3.82071591513403, |
| 4.065076420347906, |
| 4.336546325457651, |
| 4.509566349923693, |
| 4.864779841467461, |
| 5.163685002097868, |
| 5.492270513445782, |
| 5.631055430843394, |
| 5.7307215515450345, |
| 6.169248493915814, |
| 6.30570854260859, |
| 6.740466337947592, |
| 7.197941294930048, |
| 7.517601327917087, |
| 7.775720855684915 |
| ], |
| "zero": [ |
| 2.357140279396316, |
| 3.2570838792728423, |
| 4.181465995925227, |
| 4.749169230701531, |
| 5.34534413231275, |
| 6.021420744251043, |
| 6.454315443808591, |
| 6.723551631804164, |
| 7.297310461944863, |
| 7.843812972349606, |
| 8.452741835232821, |
| 9.309187261837595, |
| 9.544175996615984, |
| 10.25337440056365, |
| 11.044870113612715, |
| 11.157091535977923, |
| 11.775453001083749, |
| 12.27605148840522, |
| 12.972146691378654, |
| 13.635383275262825 |
| ] |
| } |
| }, |
| "note": "Held-out, mean over 30 segments × 3 seeds. All three variants track real motion far better than doing nothing (11.47 px at +16.7 s), but on held-out data the plain rigid-frame control drifts least." |
| }, |
| "strips": [ |
| { |
| "split": "test", |
| "label": "Held-out", |
| "src": "longterm_strip_test.png", |
| "caption": "The real frame at t0 carried forward by predicted flow alone, 16.7 s. Row 1 is what actually happened. Notice the real gel releases contact entirely by +5 s while every warped row still smears the original stripes -- the limitation is advection, not flow quality." |
| }, |
| { |
| "split": "train", |
| "label": "Seen in training", |
| "src": "longterm_strip_train.png", |
| "caption": "Same construction on an episode the models trained on." |
| } |
| ], |
| "metrics": [ |
| [ |
| "epe_act", |
| "active EPE", |
| "lower" |
| ], |
| [ |
| "epe_contact", |
| "EPE, contact frames", |
| "lower" |
| ], |
| [ |
| "epe_nocontact", |
| "EPE, no-contact frames", |
| "lower" |
| ], |
| [ |
| "cos_act", |
| "direction cos", |
| "higher" |
| ] |
| ], |
| "agg": { |
| "test": { |
| "rigid3": { |
| "epe_act": { |
| "mean": 1.1372808096509701, |
| "sd": 0.011538903769754378 |
| }, |
| "epe_contact": { |
| "mean": 1.2802531327962468, |
| "sd": 0.01449916595774911 |
| }, |
| "epe_nocontact": { |
| "mean": 0.5514139326425496, |
| "sd": 0.005381640516221394 |
| }, |
| "cos_act": { |
| "mean": 0.7191401155013454, |
| "sd": 0.007296117048381025 |
| } |
| }, |
| "gel3": { |
| "epe_act": { |
| "mean": 1.1405164966390318, |
| "sd": 0.0051759559484549815 |
| }, |
| "epe_contact": { |
| "mean": 1.2958097916254638, |
| "sd": 0.0056571786875742784 |
| }, |
| "epe_nocontact": { |
| "mean": 0.5448989845079661, |
| "sd": 0.0035435605368193453 |
| }, |
| "cos_act": { |
| "mean": 0.7239848506993065, |
| "sd": 0.00293148242545277 |
| } |
| }, |
| "gel5": { |
| "epe_act": { |
| "mean": 1.1346964696417523, |
| "sd": 0.005865391959433093 |
| }, |
| "epe_contact": { |
| "mean": 1.2913115811878275, |
| "sd": 0.011518214834769669 |
| }, |
| "epe_nocontact": { |
| "mean": 0.5383610941338166, |
| "sd": 0.0024083140093319453 |
| }, |
| "cos_act": { |
| "mean": 0.7300046482560615, |
| "sd": 0.0004607748049456351 |
| } |
| } |
| }, |
| "train": { |
| "rigid3": { |
| "epe_act": { |
| "mean": 1.168846071715971, |
| "sd": 0.02670330031534338 |
| }, |
| "epe_contact": { |
| "mean": 1.268663661820548, |
| "sd": 0.03308856615551744 |
| }, |
| "epe_nocontact": { |
| "mean": 0.5119097645784615, |
| "sd": 0.004141115765548016 |
| }, |
| "cos_act": { |
| "mean": 0.7053772315956334, |
| "sd": 0.010010152683389225 |
| } |
| }, |
| "gel3": { |
| "epe_act": { |
| "mean": 1.1519802708495182, |
| "sd": 0.013736529020491654 |
| }, |
| "epe_contact": { |
| "mean": 1.2458926085890274, |
| "sd": 0.021356951862201053 |
| }, |
| "epe_nocontact": { |
| "mean": 0.5081458158675033, |
| "sd": 0.011861079124671705 |
| }, |
| "cos_act": { |
| "mean": 0.7137595714149224, |
| "sd": 0.004672212131396673 |
| } |
| }, |
| "gel5": { |
| "epe_act": { |
| "mean": 1.1281619380450103, |
| "sd": 0.013708459123993816 |
| }, |
| "epe_contact": { |
| "mean": 1.2136604384289214, |
| "sd": 0.019016938690445146 |
| }, |
| "epe_nocontact": { |
| "mean": 0.49688398844144444, |
| "sd": 0.003875750009644446 |
| }, |
| "cos_act": { |
| "mean": 0.722527303095208, |
| "sd": 0.005426174846444434 |
| } |
| } |
| }, |
| "gap": { |
| "rigid3": { |
| "epe_act": -0.031565262065000876, |
| "epe_contact": 0.011589470975698735, |
| "epe_nocontact": 0.039504168064088074, |
| "cos_act": 0.013762883905712009 |
| }, |
| "gel3": { |
| "epe_act": -0.011463774210486388, |
| "epe_contact": 0.04991718303643644, |
| "epe_nocontact": 0.03675316864046285, |
| "cos_act": 0.010225279284384103 |
| }, |
| "gel5": { |
| "epe_act": 0.006534531596741999, |
| "epe_contact": 0.07765114275890617, |
| "epe_nocontact": 0.041477105692372174, |
| "cos_act": 0.007477345160853521 |
| } |
| } |
| }, |
| "best": { |
| "test": { |
| "epe_act": "gel5", |
| "epe_contact": "rigid3", |
| "epe_nocontact": "gel5", |
| "cos_act": "gel5" |
| }, |
| "train": { |
| "epe_act": "gel5", |
| "epe_contact": "gel5", |
| "epe_nocontact": "gel5", |
| "cos_act": "gel5" |
| } |
| }, |
| "blurb": { |
| "test": "Held-out: motherboard_0510_episode_005, motherboard_0510_episode_006. Never seen in training. Mean ± sd over 3 seeds.", |
| "train": "Seen in training: 6 episodes spanning all three recording days. Mean ± sd over 3 seeds." |
| }, |
| "panels": [ |
| { |
| "split": "test", |
| "label": "Held-out", |
| "src": "panel_test.png", |
| "caption": "Held-out episodes, seed 0. Highest-motion transitions. Columns: gel tactile frame, the action map fed in, one prediction per variant, AllTracker ground truth. Per-panel EPE in each title." |
| }, |
| { |
| "split": "train", |
| "label": "Seen in training", |
| "src": "panel_train.png", |
| "caption": "Episodes the models trained on, seed 0, same selection rule. Compare against the held-out panel: the fit is visibly tighter here." |
| } |
| ], |
| "summary": [ |
| "<strong>The flow itself is correct.</strong> Warping a held-out frame by the stored ground-truth flow beats not warping by +1.98 dB; sign flip, axis swap and single-component negation all lose, and gain/y-scale sweeps peak at 1.1 and 0.9. What the models get wrong is magnitude: cos 0.81 but only 79% of ground-truth magnitude.", |
| "<strong>You cannot get the next frame by warping the previous one here, and that is physics rather than a bug.</strong> A perfect, dense, correctly-splatted ground-truth flow explains only 13% of the change between adjacent frames. Grid resolution costs 0.12 dB and the warp operator 0.07 dB — neither is the bottleneck.", |
| "<strong>The aggregate held-out result reads flat because it is diluted.</strong> 169 of 352 held-out transitions (48%) have no contact at either end, and there all three variants are identical. In the contact regimes the gel variants beat the control on magnitude ratio every time: sustained 0.76 → 0.79, release 0.81 → 0.87, and the advantage grows with how much the video changes.", |
| "<strong>Sustained contact is the hardest regime, not the easiest</strong> — largest true motion (2.14 px), worst direction (0.68 cos) — and it is precisely what optical flow is supposed to handle.", |
| "<strong>The in-sample gain does not transfer.</strong> On episodes it trained on, gel+force takes active EPE 1.1688 → 1.1282; on held-out it is 1.1373 → 1.1347 against a seed sd of 0.0115. The train→test gap grows monotonically with what you add (-0.0316, -0.0115, +0.0065), and over 16.7 s of integration the drift ranking flips outright between splits.", |
| "<strong>The magnitude deficit is only half fixable by reweighting.</strong> Adding an explicit <code>| |pred| − |target| |</code> term moves the aggregate ratio 0.787 → 0.862 at no cost to direction, but per-cell it mostly inflates small flows: the least-squares slope moves only 0.62 → 0.65 and the largest flows stay ~35% short. The residue looks information-limited — the action does not determine how much the gel will actually move.", |
| "<strong>Next thing to try.</strong> The force channel is a flat per-frame scalar broadcast over the grid, so it carries how hard but not <em>where</em>. Modulating it by the per-cell contact mask targets exactly the regimes where it already helps." |
| ], |
| "method": [ |
| [ |
| "Task", |
| "action map + tactile latent → AllTracker flow, 16×16 latent grid" |
| ], |
| [ |
| "Model", |
| "ActionMotionEncoder, ablation D (no physics prior), ~800k params" |
| ], |
| [ |
| "Training", |
| "12 epochs, bs 64, AdamW lr 3e-4, 30 episodes, 3 seeds" |
| ], |
| [ |
| "Force", |
| "log1p(N), compressive-positive, + contact flag; k = 2.0 N/mm (assumed)" |
| ], |
| [ |
| "Gel axis", |
| "left (−0.174, −0.932, −0.317), right (+0.350, −0.925, +0.149) in rigid coords" |
| ], |
| [ |
| "Best checkpoint", |
| "lowest held-out active EPE" |
| ] |
| ], |
| "caveats": [ |
| "The published <code>amx12_D</code> (1.1298) is <em>not</em> a valid comparison: it trained on 34 episodes, these on 30 (pushT excluded, it has no force export). That is why a rigid-frame control was retrained on the same 30 episodes.", |
| "Held-out active EPE is <em>lower</em> than seen-episode EPE for the control, so the two splits differ in intrinsic difficulty. Only the change in gap <em>across variants</em> is meaningful, since all three see identical episodes.", |
| "k = 2.0 N/mm is an assumed stiffness, not a measurement. Force enters through a monotone log1p, so a wrong k is close to an affine shift the encoder absorbs.", |
| "4.8% of contact frames sit exactly at the 7.87 N ceiling; the top of the force range is censored." |
| ] |
| } |
