{ "checkpoint": "text_both_audiofix_0806 / epoch008-global_step33500", "baseline": "text_both_base_0728 / epoch034-global_step140000", "split": "test.jsonl (first 10 rows, held-out)", "cfg_scale": 1.0, "sampling_steps": 50, "staged": [ "af_add_noref", "af_add_withref", "af_remove_withref", "base_remove_noref" ], "matrix": [ { "key": "base_remove_noref", "label": "base \u00b7 remove \u00b7 noref", "model": "pre-fine-tune", "task": "remove", "ref": "noref", "in_distribution": true, "n": 10, "gap": -12.95, "right": 9, "audio": 0.617, "audio_med": 0.647 }, { "key": "base_remove_withref", "label": "base \u00b7 remove \u00b7 withref", "model": "pre-fine-tune", "task": "remove", "ref": "withref", "in_distribution": false, "n": 10, "gap": -13.18, "right": 9, "audio": 0.621, "audio_med": 0.647 }, { "key": "base_add_noref", "label": "base \u00b7 add \u00b7 noref", "model": "pre-fine-tune", "task": "add", "ref": "noref", "in_distribution": true, "n": 10, "gap": 4.71, "right": 2, "audio": 0.69, "audio_med": 0.712 }, { "key": "base_add_withref", "label": "base \u00b7 add \u00b7 withref", "model": "pre-fine-tune", "task": "add", "ref": "withref", "in_distribution": false, "n": 10, "gap": 3.23, "right": 3, "audio": 0.689, "audio_med": 0.711 }, { "key": "af_remove_noref", "label": "audiofix \u00b7 remove \u00b7 noref", "model": "audiofix", "task": "remove", "ref": "noref", "in_distribution": false, "n": 10, "gap": -7.74, "right": 7, "audio": 0.72, "audio_med": 0.854 }, { "key": "af_remove_withref", "label": "audiofix \u00b7 remove \u00b7 withref", "model": "audiofix", "task": "remove", "ref": "withref", "in_distribution": true, "n": 10, "gap": -19.35, "right": 9, "audio": 0.722, "audio_med": 0.858 }, { "key": "af_add_noref", "label": "audiofix \u00b7 add \u00b7 noref", "model": "audiofix", "task": "add", "ref": "noref", "in_distribution": false, "n": 10, "gap": 7.74, "right": 2, "audio": 0.84, "audio_med": 0.927 }, { "key": "af_add_withref", "label": "audiofix \u00b7 add \u00b7 withref", "model": "audiofix", "task": "add", "ref": "withref", "in_distribution": true, "n": 10, "gap": -2.86, "right": 6, "audio": 0.837, "audio_med": 0.925 } ], "entries": [ { "video_id": "NU2oWhlEWss", "object": "vehicles", "refs": { "input": "videos/NU2oWhlEWss_input.mp4", "target": "videos/NU2oWhlEWss_target.mp4", "pseudo_gt": "videos/NU2oWhlEWss_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -32.3, "audio": 0.823, "clip": "videos/NU2oWhlEWss_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -30.8, "audio": 0.825 }, "base_add_noref": { "gap": -1.9, "audio": 0.898 }, "base_add_withref": { "gap": -1.2, "audio": 0.897 }, "af_remove_noref": { "gap": -22.3, "audio": 0.935 }, "af_remove_withref": { "gap": -34.4, "audio": 0.932, "clip": "videos/NU2oWhlEWss_af_remove_withref.mp4" }, "af_add_noref": { "gap": 25.5, "audio": 0.921, "clip": "videos/NU2oWhlEWss_af_add_noref.mp4" }, "af_add_withref": { "gap": 16.3, "audio": 0.92, "clip": "videos/NU2oWhlEWss_af_add_withref.mp4" } } }, { "video_id": "X7t8wKNIAP4", "object": "birds", "refs": { "input": "videos/X7t8wKNIAP4_input.mp4", "target": "videos/X7t8wKNIAP4_target.mp4", "pseudo_gt": "videos/X7t8wKNIAP4_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -15.2, "audio": 0.734, "clip": "videos/X7t8wKNIAP4_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -15.8, "audio": 0.734 }, "base_add_noref": { "gap": 7.1, "audio": 0.535 }, "base_add_withref": { "gap": 6.1, "audio": 0.53 }, "af_remove_noref": { "gap": -3.4, "audio": 0.623 }, "af_remove_withref": { "gap": -18.5, "audio": 0.627, "clip": "videos/X7t8wKNIAP4_af_remove_withref.mp4" }, "af_add_noref": { "gap": 11.9, "audio": 0.736, "clip": "videos/X7t8wKNIAP4_af_add_noref.mp4" }, "af_add_withref": { "gap": 4.3, "audio": 0.728, "clip": "videos/X7t8wKNIAP4_af_add_withref.mp4" } } }, { "video_id": "Rumslm1EQhU", "object": "water", "refs": { "input": "videos/Rumslm1EQhU_input.mp4", "target": "videos/Rumslm1EQhU_target.mp4", "pseudo_gt": "videos/Rumslm1EQhU_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -25.6, "audio": 0.605, "clip": "videos/Rumslm1EQhU_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -23.6, "audio": 0.604 }, "base_add_noref": { "gap": -21.6, "audio": 0.669 }, "base_add_withref": { "gap": -26.1, "audio": 0.668 }, "af_remove_noref": { "gap": -30.5, "audio": 0.962 }, "af_remove_withref": { "gap": -35.5, "audio": 0.962, "clip": "videos/Rumslm1EQhU_af_remove_withref.mp4" }, "af_add_noref": { "gap": -23.2, "audio": 0.965, "clip": "videos/Rumslm1EQhU_af_add_noref.mp4" }, "af_add_withref": { "gap": -30.7, "audio": 0.965, "clip": "videos/Rumslm1EQhU_af_add_withref.mp4" } } }, { "video_id": "65vCgp9aKGU_000116", "object": "birds", "refs": { "input": "videos/65vCgp9aKGU_000116_input.mp4", "target": "videos/65vCgp9aKGU_000116_target.mp4", "pseudo_gt": "videos/65vCgp9aKGU_000116_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -11.6, "audio": 0.558, "clip": "videos/65vCgp9aKGU_000116_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -13.2, "audio": 0.557 }, "base_add_noref": { "gap": 11.2, "audio": 0.558 }, "base_add_withref": { "gap": 11.5, "audio": 0.559 }, "af_remove_noref": { "gap": -7.2, "audio": 0.871 }, "af_remove_withref": { "gap": -19.2, "audio": 0.874, "clip": "videos/65vCgp9aKGU_000116_af_remove_withref.mp4" }, "af_add_noref": { "gap": 13.6, "audio": 0.875, "clip": "videos/65vCgp9aKGU_000116_af_add_noref.mp4" }, "af_add_withref": { "gap": 9.0, "audio": 0.87, "clip": "videos/65vCgp9aKGU_000116_af_add_withref.mp4" } } }, { "video_id": "hVnvazN3U0s_000060", "object": "motorboat engine", "refs": { "input": "videos/hVnvazN3U0s_000060_input.mp4", "target": "videos/hVnvazN3U0s_000060_target.mp4", "pseudo_gt": "videos/hVnvazN3U0s_000060_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -19.8, "audio": 0.678, "clip": "videos/hVnvazN3U0s_000060_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -20.1, "audio": 0.675 }, "base_add_noref": { "gap": 19.2, "audio": 0.754 }, "base_add_withref": { "gap": 11.7, "audio": 0.754 }, "af_remove_noref": { "gap": -18.1, "audio": 0.837 }, "af_remove_withref": { "gap": -21.0, "audio": 0.805, "clip": "videos/hVnvazN3U0s_000060_af_remove_withref.mp4" }, "af_add_noref": { "gap": 13.7, "audio": 0.97, "clip": "videos/hVnvazN3U0s_000060_af_add_noref.mp4" }, "af_add_withref": { "gap": -11.7, "audio": 0.97, "clip": "videos/hVnvazN3U0s_000060_af_add_withref.mp4" } } }, { "video_id": "u_8JdjQOo-o_000151", "object": "bagpipe", "refs": { "input": "videos/u_8JdjQOo-o_000151_input.mp4", "target": "videos/u_8JdjQOo-o_000151_target.mp4", "pseudo_gt": "videos/u_8JdjQOo-o_000151_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -4.3, "audio": 0.596, "clip": "videos/u_8JdjQOo-o_000151_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -2.7, "audio": 0.595 }, "base_add_noref": { "gap": 5.5, "audio": 0.35 }, "base_add_withref": { "gap": 7.1, "audio": 0.344 }, "af_remove_noref": { "gap": 6.7, "audio": 0.362 }, "af_remove_withref": { "gap": -13.3, "audio": 0.369, "clip": "videos/u_8JdjQOo-o_000151_af_remove_withref.mp4" }, "af_add_noref": { "gap": 13.6, "audio": 0.198, "clip": "videos/u_8JdjQOo-o_000151_af_add_noref.mp4" }, "af_add_withref": { "gap": -3.1, "audio": 0.192, "clip": "videos/u_8JdjQOo-o_000151_af_add_withref.mp4" } } }, { "video_id": "TbTUbDC-IuQ_000249", "object": "bass line", "refs": { "input": "videos/TbTUbDC-IuQ_000249_input.mp4", "target": "videos/TbTUbDC-IuQ_000249_target.mp4", "pseudo_gt": "videos/TbTUbDC-IuQ_000249_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -9.4, "audio": 0.724, "clip": "videos/TbTUbDC-IuQ_000249_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -11.4, "audio": 0.724 }, "base_add_noref": { "gap": 9.3, "audio": 0.662 }, "base_add_withref": { "gap": 11.8, "audio": 0.664 }, "af_remove_noref": { "gap": -4.6, "audio": 0.945 }, "af_remove_withref": { "gap": -25.1, "audio": 0.945, "clip": "videos/TbTUbDC-IuQ_000249_af_remove_withref.mp4" }, "af_add_noref": { "gap": 17.3, "audio": 0.933, "clip": "videos/TbTUbDC-IuQ_000249_af_add_noref.mp4" }, "af_add_withref": { "gap": 10.3, "audio": 0.929, "clip": "videos/TbTUbDC-IuQ_000249_af_add_withref.mp4" } } }, { "video_id": "GwA7hMcmJ6Q", "object": "piano", "refs": { "input": "videos/GwA7hMcmJ6Q_input.mp4", "target": "videos/GwA7hMcmJ6Q_target.mp4", "pseudo_gt": "videos/GwA7hMcmJ6Q_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -5.7, "audio": 0.685, "clip": "videos/GwA7hMcmJ6Q_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -10.7, "audio": 0.717 }, "base_add_noref": { "gap": 5.0, "audio": 0.824 }, "base_add_withref": { "gap": 3.1, "audio": 0.835 }, "af_remove_noref": { "gap": 1.6, "audio": -0.022 }, "af_remove_withref": { "gap": -17.3, "audio": -0.02, "clip": "videos/GwA7hMcmJ6Q_af_remove_withref.mp4" }, "af_add_noref": { "gap": 5.2, "audio": 0.963, "clip": "videos/GwA7hMcmJ6Q_af_add_noref.mp4" }, "af_add_withref": { "gap": -10.5, "audio": 0.961, "clip": "videos/GwA7hMcmJ6Q_af_add_withref.mp4" } } }, { "video_id": "884QLkT8sBQ_000098", "object": "musician", "refs": { "input": "videos/884QLkT8sBQ_000098_input.mp4", "target": "videos/884QLkT8sBQ_000098_target.mp4", "pseudo_gt": "videos/884QLkT8sBQ_000098_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": -12.8, "audio": 0.615, "clip": "videos/884QLkT8sBQ_000098_base_remove_noref.mp4" }, "base_remove_withref": { "gap": -12.5, "audio": 0.62 }, "base_add_noref": { "gap": 2.2, "audio": 0.774 }, "base_add_withref": { "gap": -2.5, "audio": 0.772 }, "af_remove_noref": { "gap": -7.8, "audio": 0.88 }, "af_remove_withref": { "gap": -16.9, "audio": 0.883, "clip": "videos/884QLkT8sBQ_000098_af_remove_withref.mp4" }, "af_add_noref": { "gap": 5.5, "audio": 0.9, "clip": "videos/884QLkT8sBQ_000098_af_add_noref.mp4" }, "af_add_withref": { "gap": -3.7, "audio": 0.898, "clip": "videos/884QLkT8sBQ_000098_af_add_withref.mp4" } } }, { "video_id": "NhWaVKzBX_g_000056", "object": "powerful engine", "refs": { "input": "videos/NhWaVKzBX_g_000056_input.mp4", "target": "videos/NhWaVKzBX_g_000056_target.mp4", "pseudo_gt": "videos/NhWaVKzBX_g_000056_pseudo_gt.mp4" }, "cells": { "base_remove_noref": { "gap": 7.2, "audio": 0.148, "clip": "videos/NhWaVKzBX_g_000056_base_remove_noref.mp4" }, "base_remove_withref": { "gap": 9.0, "audio": 0.162 }, "base_add_noref": { "gap": 11.1, "audio": 0.872 }, "base_add_withref": { "gap": 10.8, "audio": 0.868 }, "af_remove_noref": { "gap": 8.2, "audio": 0.804 }, "af_remove_withref": { "gap": 7.7, "audio": 0.842, "clip": "videos/NhWaVKzBX_g_000056_af_remove_withref.mp4" }, "af_add_noref": { "gap": -5.7, "audio": 0.936, "clip": "videos/NhWaVKzBX_g_000056_af_add_noref.mp4" }, "af_add_withref": { "gap": -8.8, "audio": 0.935, "clip": "videos/NhWaVKzBX_g_000056_af_add_withref.mp4" } } } ], "motion": { "config": { "num_points": 64, "sampling": "hybrid", "flow_unit": 8, "alpha_frac": 0.5, "steps": 25, "cfg": 7.0, "resolution": [ 320, 320 ], "model_length": 16, "stride": [ 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120 ], "null_prompt": false, "seeds": [ 42, 1, 7 ] }, "aggregate": { "epe_all": { "mean": 24.658, "median": 26.295 }, "epe_move": { "mean": 26.9843, "median": 29.3846 }, "epe_hint": { "mean": 24.5736, "median": 26.6967 }, "epe_static": { "mean": 13.6612, "median": 8.5039 }, "cos": { "mean": 0.7623, "median": 0.7587 }, "mag_ratio": { "mean": 1.3092, "median": 1.0649 }, "epe_rel": { "mean": 0.8741, "median": 0.6953 }, "gt_mean_mag": { "mean": 36.7063, "median": 40.2252 }, "moving_frac": { "mean": 0.4602, "median": 0.4624 }, "epe_hint_input": { "mean": 3.0596, "median": 2.1739 }, "vae_floor_epe_all": { "mean": 5.1819, "median": 4.282 }, "vae_floor_epe_move": { "mean": 5.2658, "median": 4.3934 }, "epe_over_floor": { "mean": 5.442, "median": 5.0685 }, "n_seeds": { "mean": 63.2667, "median": 64.0 }, "hint_coverage": { "mean": 0.1488, "median": 0.1494 }, "epe_move_spread": { "mean": 9.5138, "median": 7.0769 }, "cos_spread": { "mean": 0.1577, "median": 0.1458 }, "epe_hint_spread": { "mean": 11.9112, "median": 10.3256 }, "median_across_seeds": { "values": [ 33.77, 22.26, 26.01 ], "range": 11.51 } }, "per_clip": [ { "video_id": "NU2oWhlEWss", "object": "vehicles", "prompt": "vehicles", "epe_all": 32.1522, "epe_all_spread": 12.3304, "epe_move": 32.1623, "epe_move_spread": 12.3324, "epe_hint": 26.2805, "epe_hint_spread": 11.1704, "epe_static": 8.6428, "epe_static_spread": 6.4733, "cos": 0.8564, "cos_spread": 0.1507, "mag_ratio": 0.83, "mag_ratio_spread": 0.2017, "epe_rel": 0.5103, "epe_rel_spread": 0.1869, "gt_mean_mag": 62.9454, "gt_mean_mag_spread": 1.0894, "moving_frac": 0.3998, "moving_frac_spread": 0.0144, "epe_hint_input": 2.5554, "epe_hint_input_spread": 0.3789, "vae_floor_epe_all": 8.0003, "vae_floor_epe_all_spread": 0.6853, "vae_floor_epe_move": 7.9999, "vae_floor_epe_move_spread": 0.6851, "epe_over_floor": 4.0427, "epe_over_floor_spread": 1.8748, "n_seeds": 61.0, "n_seeds_spread": 0, "hint_coverage": 0.1437, "hint_coverage_spread": 0.0025, "clip": "videos/flow_NU2oWhlEWss_roundtrip.mp4" }, { "video_id": "X7t8wKNIAP4", "object": "birds", "prompt": "birds", "epe_all": 27.879, "epe_all_spread": 10.8634, "epe_move": 28.0293, "epe_move_spread": 10.9683, "epe_hint": 30.4758, "epe_hint_spread": 10.765, "epe_static": 5.6995, "epe_static_spread": 4.0561, "cos": 0.7341, "cos_spread": 0.1492, "mag_ratio": 1.5515, "mag_ratio_spread": 0.4401, "epe_rel": 0.8792, "epe_rel_spread": 0.3583, "gt_mean_mag": 31.7001, "gt_mean_mag_spread": 0.59, "moving_frac": 0.6556, "moving_frac_spread": 0.0064, "epe_hint_input": 1.7641, "epe_hint_input_spread": 0.6358, "vae_floor_epe_all": 3.392, "vae_floor_epe_all_spread": 0.0379, "vae_floor_epe_move": 3.3967, "vae_floor_epe_move_spread": 0.0369, "epe_over_floor": 8.2456, "epe_over_floor_spread": 3.1323, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1527, "hint_coverage_spread": 0.0106, "clip": "videos/flow_X7t8wKNIAP4_roundtrip.mp4" }, { "video_id": "Rumslm1EQhU", "object": "water", "prompt": "water", "epe_all": 33.1671, "epe_all_spread": 3.0465, "epe_move": 33.1983, "epe_move_spread": 2.9564, "epe_hint": 32.1478, "epe_hint_spread": 6.1577, "epe_static": 9.5747, "epe_static_spread": 8.473, "cos": 0.8977, "cos_spread": 0.004, "mag_ratio": 0.9375, "mag_ratio_spread": 0.2757, "epe_rel": 0.5197, "epe_rel_spread": 0.0639, "gt_mean_mag": 64.0422, "gt_mean_mag_spread": 11.8828, "moving_frac": 0.1493, "moving_frac_spread": 0.03, "epe_hint_input": 5.0858, "epe_hint_input_spread": 1.283, "vae_floor_epe_all": 5.0789, "vae_floor_epe_all_spread": 1.3791, "vae_floor_epe_move": 5.0811, "vae_floor_epe_move_spread": 1.3727, "epe_over_floor": 6.5883, "epe_over_floor_spread": 1.1571, "n_seeds": 60.3333, "n_seeds_spread": 7, "hint_coverage": 0.1415, "hint_coverage_spread": 0.02, "clip": "videos/flow_Rumslm1EQhU_roundtrip.mp4" }, { "video_id": "65vCgp9aKGU_000116", "object": "birds", "prompt": "birds", "epe_all": 31.2466, "epe_all_spread": 34.6292, "epe_move": 30.74, "epe_move_spread": 34.4406, "epe_hint": 30.1823, "epe_hint_spread": 36.3044, "epe_static": 54.0218, "epe_static_spread": 65.5342, "cos": 0.839, "cos_spread": 0.2792, "mag_ratio": 1.2533, "mag_ratio_spread": 1.2906, "epe_rel": 0.6185, "epe_rel_spread": 0.701, "gt_mean_mag": 48.7503, "gt_mean_mag_spread": 0.7672, "moving_frac": 0.7019, "moving_frac_spread": 0.0057, "epe_hint_input": 1.6455, "epe_hint_input_spread": 0.715, "vae_floor_epe_all": 3.6292, "vae_floor_epe_all_spread": 0.7059, "vae_floor_epe_move": 3.603, "vae_floor_epe_move_spread": 0.7425, "epe_over_floor": 8.5302, "epe_over_floor_spread": 9.2405, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1483, "hint_coverage_spread": 0.0088, "clip": "videos/flow_65vCgp9aKGU_000116_roundtrip.mp4" }, { "video_id": "hVnvazN3U0s_000060", "object": "motorboat engine", "prompt": "motorboat engine", "epe_all": 14.6474, "epe_all_spread": 1.8732, "epe_move": 16.9112, "epe_move_spread": 3.7384, "epe_hint": 15.0685, "epe_hint_spread": 2.3115, "epe_static": 9.1301, "epe_static_spread": 4.3417, "cos": 0.6685, "cos_spread": 0.1305, "mag_ratio": 1.0705, "mag_ratio_spread": 0.1108, "epe_rel": 0.6461, "epe_rel_spread": 0.1402, "gt_mean_mag": 18.7568, "gt_mean_mag_spread": 0.9627, "moving_frac": 0.525, "moving_frac_spread": 0.0212, "epe_hint_input": 1.7924, "epe_hint_input_spread": 0.3748, "vae_floor_epe_all": 3.6299, "vae_floor_epe_all_spread": 0.0598, "vae_floor_epe_move": 3.9292, "vae_floor_epe_move_spread": 0.0338, "epe_over_floor": 4.3049, "epe_over_floor_spread": 0.9774, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1504, "hint_coverage_spread": 0.0012, "clip": "videos/flow_hVnvazN3U0s_000060_roundtrip.mp4" }, { "video_id": "u_8JdjQOo-o_000151", "object": "bagpipe", "prompt": "bagpipe", "epe_all": 13.0996, "epe_all_spread": 8.4247, "epe_move": 13.6013, "epe_move_spread": 8.6718, "epe_hint": 14.9405, "epe_hint_spread": 9.8863, "epe_static": 7.5495, "epe_static_spread": 6.1757, "cos": 0.7191, "cos_spread": 0.1372, "mag_ratio": 2.2583, "mag_ratio_spread": 1.0886, "epe_rel": 1.5735, "epe_rel_spread": 1.0417, "gt_mean_mag": 8.0127, "gt_mean_mag_spread": 0.2352, "moving_frac": 0.8434, "moving_frac_spread": 0.0092, "epe_hint_input": 0.8816, "epe_hint_input_spread": 0.0984, "vae_floor_epe_all": 4.0564, "vae_floor_epe_all_spread": 0.0243, "vae_floor_epe_move": 4.0651, "vae_floor_epe_move_spread": 0.0255, "epe_over_floor": 3.3442, "epe_over_floor_spread": 2.1119, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.144, "hint_coverage_spread": 0.0012, "clip": "videos/flow_u_8JdjQOo-o_000151_roundtrip.mp4" }, { "video_id": "TbTUbDC-IuQ_000249", "object": "bass line", "prompt": "bass line", "epe_all": 40.7904, "epe_all_spread": 5.7336, "epe_move": 41.169, "epe_move_spread": 5.4819, "epe_hint": 38.8292, "epe_hint_spread": 15.2289, "epe_static": 21.0423, "epe_static_spread": 19.5397, "cos": 0.5953, "cos_spread": 0.2883, "mag_ratio": 0.7542, "mag_ratio_spread": 0.2489, "epe_rel": 0.7445, "epe_rel_spread": 0.1151, "gt_mean_mag": 54.2967, "gt_mean_mag_spread": 1.2008, "moving_frac": 0.2272, "moving_frac_spread": 0.0007, "epe_hint_input": 6.4574, "epe_hint_input_spread": 0.2559, "vae_floor_epe_all": 11.0431, "vae_floor_epe_all_spread": 0.978, "vae_floor_epe_move": 10.5494, "vae_floor_epe_move_spread": 0.9546, "epe_over_floor": 3.9128, "epe_over_floor_spread": 0.7345, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1548, "hint_coverage_spread": 0.005, "clip": "videos/flow_TbTUbDC-IuQ_000249_roundtrip.mp4" }, { "video_id": "GwA7hMcmJ6Q", "object": "piano", "prompt": "piano", "epe_all": 24.7111, "epe_all_spread": 8.8532, "epe_move": 24.7477, "epe_move_spread": 8.8167, "epe_hint": 24.1111, "epe_hint_spread": 9.421, "epe_static": 5.7746, "epe_static_spread": 2.869, "cos": 0.8921, "cos_spread": 0.0853, "mag_ratio": 0.953, "mag_ratio_spread": 0.3776, "epe_rel": 0.4365, "epe_rel_spread": 0.1186, "gt_mean_mag": 56.346, "gt_mean_mag_spread": 5.288, "moving_frac": 0.5425, "moving_frac_spread": 0.0529, "epe_hint_input": 2.6156, "epe_hint_input_spread": 0.3084, "vae_floor_epe_all": 4.2329, "vae_floor_epe_all_spread": 0.1649, "vae_floor_epe_move": 4.2342, "vae_floor_epe_move_spread": 0.1629, "epe_over_floor": 5.8321, "epe_over_floor_spread": 1.9388, "n_seeds": 63.3333, "n_seeds_spread": 1, "hint_coverage": 0.1533, "hint_coverage_spread": 0.01, "clip": "videos/flow_GwA7hMcmJ6Q_roundtrip.mp4" }, { "video_id": "884QLkT8sBQ_000098", "object": "musician", "prompt": "musician", "epe_all": 7.1062, "epe_all_spread": 5.047, "epe_move": 7.6871, "epe_move_spread": 4.2531, "epe_hint": 6.5878, "epe_hint_spread": 6.1963, "epe_static": 6.8113, "epe_static_spread": 5.483, "cos": 0.6377, "cos_spread": 0.1424, "mag_ratio": 2.4247, "mag_ratio_spread": 0.7748, "epe_rel": 1.9894, "epe_rel_spread": 1.0724, "gt_mean_mag": 1.5478, "gt_mean_mag_spread": 0.1048, "moving_frac": 0.2948, "moving_frac_spread": 0.0229, "epe_hint_input": 0.7487, "epe_hint_input_spread": 0.293, "vae_floor_epe_all": 4.4254, "vae_floor_epe_all_spread": 0.0341, "vae_floor_epe_move": 4.5526, "vae_floor_epe_move_spread": 0.0568, "epe_over_floor": 1.6868, "epe_over_floor_spread": 0.9133, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1517, "hint_coverage_spread": 0.0063, "clip": "videos/flow_884QLkT8sBQ_000098_roundtrip.mp4" }, { "video_id": "NhWaVKzBX_g_000056", "object": "powerful engine", "prompt": "powerful engine", "epe_all": 21.7806, "epe_all_spread": 5.4538, "epe_move": 41.5973, "epe_move_spread": 3.4781, "epe_hint": 27.1129, "epe_hint_spread": 11.6709, "epe_static": 8.365, "epe_static_spread": 8.1424, "cos": 0.7832, "cos_spread": 0.2101, "mag_ratio": 1.0593, "mag_ratio_spread": 0.3306, "epe_rel": 0.8231, "epe_rel_spread": 0.0582, "gt_mean_mag": 20.6647, "gt_mean_mag_spread": 0.7411, "moving_frac": 0.2627, "moving_frac_spread": 0.0347, "epe_hint_input": 7.0493, "epe_hint_input_spread": 2.5335, "vae_floor_epe_all": 4.3311, "vae_floor_epe_all_spread": 0.0803, "vae_floor_epe_move": 5.2465, "vae_floor_epe_move_spread": 0.5302, "epe_over_floor": 7.9323, "epe_over_floor_spread": 0.5097, "n_seeds": 64.0, "n_seeds_spread": 0, "hint_coverage": 0.1477, "hint_coverage_spread": 0.0069, "clip": "videos/flow_NhWaVKzBX_g_000056_roundtrip.mp4" } ], "drawn": { "flow_unit": 64, "seed": 42, "groups": [ "direction", "speed", "path shape", "many drags", "independent", "brush" ], "scenarios": [ { "group": "direction", "name": "drag right", "clip": "videos/drawn_direction_drag_right.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 51.14, "max_flow": 253.59 }, { "group": "direction", "name": "drag left", "clip": "videos/drawn_direction_drag_left.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 26.33, "max_flow": 163.78 }, { "group": "direction", "name": "drag down (toward camera)", "clip": "videos/drawn_direction_drag_down_toward_camera.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 16.44, "max_flow": 126.03 }, { "group": "speed", "name": "short line (~30 px)", "clip": "videos/drawn_speed_short_line_30_px.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 18.61, "max_flow": 77.98 }, { "group": "speed", "name": "long line (~180 px)", "clip": "videos/drawn_speed_long_line_180_px.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 53.39, "max_flow": 261.42 }, { "group": "path shape", "name": "straight line", "clip": "videos/drawn_path_shape_straight_line.mp4", "image": "Messi-is-shooting.jpg", "prompt": "slow motion, a football player is shooting, the ball is flying", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 96.31, "max_flow": 373.2 }, { "group": "path shape", "name": "arc over the same span", "clip": "videos/drawn_path_shape_arc_over_the_same_span.mp4", "image": "Messi-is-shooting.jpg", "prompt": "slow motion, a football player is shooting, the ball is flying", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 110.07, "max_flow": 374.73 }, { "group": "many drags", "name": "zoom in (4 drags outward)", "clip": "videos/drawn_many_drags_zoom_in_4_drags_outward.mp4", "image": "landscape.jpg", "prompt": "aerial footage flying over a pine forest", "n_drags": 4, "brush": false, "coverage": 0.423, "mean_flow": 16.43, "max_flow": 94.32 }, { "group": "many drags", "name": "zoom out (4 drags inward)", "clip": "videos/drawn_many_drags_zoom_out_4_drags_inward.mp4", "image": "landscape.jpg", "prompt": "aerial footage flying over a pine forest", "n_drags": 4, "brush": false, "coverage": 0.64, "mean_flow": 29.2, "max_flow": 128.24 }, { "group": "many drags", "name": "rotate (4 tangential drags)", "clip": "videos/drawn_many_drags_rotate_4_tangential_drags.mp4", "image": "landscape.jpg", "prompt": "aerial footage flying over a pine forest", "n_drags": 4, "brush": false, "coverage": 0.55, "mean_flow": 28.13, "max_flow": 106.5 }, { "group": "many drags", "name": "pan right (3 drags)", "clip": "videos/drawn_many_drags_pan_right_3_drags.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 3, "brush": false, "coverage": 0.467, "mean_flow": 24.3, "max_flow": 114.47 }, { "group": "independent", "name": "two birds, opposite directions", "clip": "videos/drawn_independent_two_birds_opposite_directions.mp4", "image": "1714.jpg", "prompt": "a flock of seagulls flying over the water", "n_drags": 2, "brush": false, "coverage": 0.265, "mean_flow": 31.73, "max_flow": 110.26 }, { "group": "brush", "name": "no brush: the whole scene moves", "clip": "videos/drawn_brush_no_brush_the_whole_scene_moves.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": false, "coverage": 0.16, "mean_flow": 51.14, "max_flow": 253.59 }, { "group": "brush", "name": "brush on the car only", "clip": "videos/drawn_brush_brush_on_the_car_only.mp4", "image": "photo-of-fast-moving-red-car.jpg", "prompt": "a fast driving red car, close up, slow motion", "n_drags": 1, "brush": true, "coverage": 0.877, "mean_flow": 8.74, "max_flow": 142.01 } ] }, "object": { "config": { "model": "auto", "tau": 2.0, "seed": 42, "resolution": [ 320, 320 ] }, "aggregate": { "mean_total": 63.03, "mean_camera": 35.97, "mean_object": 21.065, "camera_share": 0.451, "inlier_frac": 0.546, "moving_frac": 0.237 }, "per_clip": [ { "video_id": "NU2oWhlEWss", "object": "vehicles", "clip": "videos/object_NU2oWhlEWss.mp4", "mean_total": 161.89, "mean_camera": 54.37, "mean_object": 130.17, "camera_share": 0.336, "inlier_frac": 0.562, "moving_frac": 0.261 }, { "video_id": "X7t8wKNIAP4", "object": "birds", "clip": "videos/object_X7t8wKNIAP4.mp4", "mean_total": 38.27, "mean_camera": 31.87, "mean_object": 8.88, "camera_share": 0.833, "inlier_frac": 0.718, "moving_frac": 0.156 }, { "video_id": "Rumslm1EQhU", "object": "water", "clip": "videos/object_Rumslm1EQhU.mp4", "mean_total": 116.96, "mean_camera": 40.07, "mean_object": 112.78, "camera_share": 0.343, "inlier_frac": 0.303, "moving_frac": 0.099 }, { "video_id": "65vCgp9aKGU_000116", "object": "birds", "clip": "videos/object_65vCgp9aKGU_000116.mp4", "mean_total": 57.23, "mean_camera": 57.25, "mean_object": 7.53, "camera_share": 1.0, "inlier_frac": 0.529, "moving_frac": 0.303 }, { "video_id": "hVnvazN3U0s_000060", "object": "motorboat engine", "clip": "videos/object_hVnvazN3U0s_000060.mp4", "mean_total": 30.37, "mean_camera": 26.06, "mean_object": 26.12, "camera_share": 0.858, "inlier_frac": 0.525, "moving_frac": 0.6 }, { "video_id": "u_8JdjQOo-o_000151", "object": "bagpipe", "clip": "videos/object_u_8JdjQOo-o_000151.mp4", "mean_total": 8.53, "mean_camera": 4.77, "mean_object": 5.21, "camera_share": 0.559, "inlier_frac": 0.646, "moving_frac": 0.31 }, { "video_id": "TbTUbDC-IuQ_000249", "object": "bass line", "clip": "videos/object_TbTUbDC-IuQ_000249.mp4", "mean_total": 233.86, "mean_camera": 53.54, "mean_object": 206.01, "camera_share": 0.229, "inlier_frac": 0.519, "moving_frac": 0.206 }, { "video_id": "GwA7hMcmJ6Q", "object": "piano", "clip": "videos/object_GwA7hMcmJ6Q.mp4", "mean_total": 68.97, "mean_camera": 75.4, "mean_object": 16.01, "camera_share": 1.093, "inlier_frac": 0.276, "moving_frac": 0.37 }, { "video_id": "884QLkT8sBQ_000098", "object": "musician", "clip": "videos/object_884QLkT8sBQ_000098.mp4", "mean_total": 2.92, "mean_camera": 0.97, "mean_object": 2.43, "camera_share": 0.333, "inlier_frac": 0.877, "moving_frac": 0.088 }, { "video_id": "NhWaVKzBX_g_000056", "object": "powerful engine", "clip": "videos/object_NhWaVKzBX_g_000056.mp4", "mean_total": 68.83, "mean_camera": 0.77, "mean_object": 68.69, "camera_share": 0.011, "inlier_frac": 0.763, "moving_frac": 0.214 } ] } } }