{ "run_id": "text_both_base_0728", "checkpoint": "epoch004-global_step17000", "sampling_steps": 50, "main_cfg": 1.0, "splits": [ { "key": "val", "label": "val.jsonl \u2014 held out", "entries": [ { "video_id": "f1w8_fTIZZE_000036", "object": "trumpets", "refs": { "add": { "source": "videos/val_f1w8_fTIZZE_000036_add_source.mp4", "target": "videos/val_f1w8_fTIZZE_000036_add_target.mp4" }, "remove": { "source": "videos/val_f1w8_fTIZZE_000036_remove_source.mp4", "target": "videos/val_f1w8_fTIZZE_000036_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with trumpets", "generated": "videos/val_f1w8_fTIZZE_000036_withref_add.mp4", "metrics": { "l1_tgt": 39.9, "l1_src": 27.4, "std": 75.6, "gt_std": 74.4, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with trumpets", "generated": "videos/val_f1w8_fTIZZE_000036_noref_add.mp4", "metrics": { "l1_tgt": 36.2, "l1_src": 30.5, "std": 75.7, "gt_std": 74.4, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without trumpets", "generated": "videos/val_f1w8_fTIZZE_000036_withref_remove.mp4", "metrics": { "l1_tgt": 29.2, "l1_src": 41.8, "std": 62.4, "gt_std": 61.8, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without trumpets", "generated": "videos/val_f1w8_fTIZZE_000036_noref_remove.mp4", "metrics": { "l1_tgt": 30.7, "l1_src": 33.3, "std": 64.4, "gt_std": 61.8, "audio": true } } ] }, { "video_id": "gDfj4eERZ6Q_000200", "object": "man", "refs": { "add": { "source": "videos/val_gDfj4eERZ6Q_000200_add_source.mp4", "target": "videos/val_gDfj4eERZ6Q_000200_add_target.mp4" }, "remove": { "source": "videos/val_gDfj4eERZ6Q_000200_remove_source.mp4", "target": "videos/val_gDfj4eERZ6Q_000200_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with man", "generated": "videos/val_gDfj4eERZ6Q_000200_withref_add.mp4", "metrics": { "l1_tgt": 22.4, "l1_src": 18.4, "std": 55.6, "gt_std": 52.5, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with man", "generated": "videos/val_gDfj4eERZ6Q_000200_noref_add.mp4", "metrics": { "l1_tgt": 22.9, "l1_src": 18.7, "std": 54.0, "gt_std": 52.5, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without man", "generated": "videos/val_gDfj4eERZ6Q_000200_withref_remove.mp4", "metrics": { "l1_tgt": 26.4, "l1_src": 30.9, "std": 35.8, "gt_std": 51.0, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without man", "generated": "videos/val_gDfj4eERZ6Q_000200_noref_remove.mp4", "metrics": { "l1_tgt": 14.3, "l1_src": 16.1, "std": 48.3, "gt_std": 51.0, "audio": true } } ] }, { "video_id": "qWfbM-MKkgs", "object": "train engine", "refs": { "add": { "source": "videos/val_qWfbM-MKkgs_add_source.mp4", "target": "videos/val_qWfbM-MKkgs_add_target.mp4" }, "remove": { "source": "videos/val_qWfbM-MKkgs_remove_source.mp4", "target": "videos/val_qWfbM-MKkgs_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with train engine", "generated": "videos/val_qWfbM-MKkgs_withref_add.mp4", "metrics": { "l1_tgt": 34.2, "l1_src": 33.2, "std": 68.7, "gt_std": 70.8, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with train engine", "generated": "videos/val_qWfbM-MKkgs_noref_add.mp4", "metrics": { "l1_tgt": 37.4, "l1_src": 33.7, "std": 66.5, "gt_std": 70.8, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without train engine", "generated": "videos/val_qWfbM-MKkgs_withref_remove.mp4", "metrics": { "l1_tgt": 31.2, "l1_src": 33.8, "std": 64.8, "gt_std": 59.6, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without train engine", "generated": "videos/val_qWfbM-MKkgs_noref_remove.mp4", "metrics": { "l1_tgt": 38.0, "l1_src": 38.0, "std": 66.0, "gt_std": 59.6, "audio": true } } ] }, { "video_id": "20qZLse0acs_000030", "object": "dog", "refs": { "add": { "source": "videos/val_20qZLse0acs_000030_add_source.mp4", "target": "videos/val_20qZLse0acs_000030_add_target.mp4" }, "remove": { "source": "videos/val_20qZLse0acs_000030_remove_source.mp4", "target": "videos/val_20qZLse0acs_000030_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with dog", "generated": "videos/val_20qZLse0acs_000030_withref_add.mp4", "metrics": { "l1_tgt": 36.4, "l1_src": 18.4, "std": 67.3, "gt_std": 75.4, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with dog", "generated": "videos/val_20qZLse0acs_000030_noref_add.mp4", "metrics": { "l1_tgt": 27.0, "l1_src": 14.7, "std": 64.2, "gt_std": 75.4, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without dog", "generated": "videos/val_20qZLse0acs_000030_withref_remove.mp4", "metrics": { "l1_tgt": 10.5, "l1_src": 26.9, "std": 63.0, "gt_std": 62.7, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without dog", "generated": "videos/val_20qZLse0acs_000030_noref_remove.mp4", "metrics": { "l1_tgt": 14.2, "l1_src": 18.3, "std": 66.3, "gt_std": 62.7, "audio": true } } ] }, { "video_id": "vo3Vx7SxaNw_000030", "object": "birds", "refs": { "add": { "source": "videos/val_vo3Vx7SxaNw_000030_add_source.mp4", "target": "videos/val_vo3Vx7SxaNw_000030_add_target.mp4" }, "remove": { "source": "videos/val_vo3Vx7SxaNw_000030_remove_source.mp4", "target": "videos/val_vo3Vx7SxaNw_000030_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with birds", "generated": "videos/val_vo3Vx7SxaNw_000030_withref_add.mp4", "metrics": { "l1_tgt": 48.6, "l1_src": 35.6, "std": 56.6, "gt_std": 65.7, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with birds", "generated": "videos/val_vo3Vx7SxaNw_000030_noref_add.mp4", "metrics": { "l1_tgt": 44.9, "l1_src": 29.6, "std": 49.6, "gt_std": 65.7, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without birds", "generated": "videos/val_vo3Vx7SxaNw_000030_withref_remove.mp4", "metrics": { "l1_tgt": 24.0, "l1_src": 45.9, "std": 38.7, "gt_std": 37.0, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without birds", "generated": "videos/val_vo3Vx7SxaNw_000030_noref_remove.mp4", "metrics": { "l1_tgt": 23.2, "l1_src": 46.4, "std": 39.3, "gt_std": 37.0, "audio": true } } ] } ] }, { "key": "train", "label": "train.jsonl \u2014 seen in training", "entries": [ { "video_id": "ITc_gC-oIUc_000064", "object": "mandolin", "refs": { "add": { "source": "videos/train_ITc_gC-oIUc_000064_add_source.mp4", "target": "videos/train_ITc_gC-oIUc_000064_add_target.mp4" }, "remove": { "source": "videos/train_ITc_gC-oIUc_000064_remove_source.mp4", "target": "videos/train_ITc_gC-oIUc_000064_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with mandolin", "generated": "videos/train_ITc_gC-oIUc_000064_withref_add.mp4", "metrics": { "l1_tgt": 38.4, "l1_src": 29.6, "std": 76.2, "gt_std": 76.0, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with mandolin", "generated": "videos/train_ITc_gC-oIUc_000064_noref_add.mp4", "metrics": { "l1_tgt": 37.1, "l1_src": 27.9, "std": 80.4, "gt_std": 76.0, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without mandolin", "generated": "videos/train_ITc_gC-oIUc_000064_withref_remove.mp4", "metrics": { "l1_tgt": 32.8, "l1_src": 30.0, "std": 71.7, "gt_std": 68.5, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without mandolin", "generated": "videos/train_ITc_gC-oIUc_000064_noref_remove.mp4", "metrics": { "l1_tgt": 19.2, "l1_src": 24.5, "std": 72.5, "gt_std": 68.5, "audio": true } } ] }, { "video_id": "PVp8gPBB6SE", "object": "cries", "refs": { "add": { "source": "videos/train_PVp8gPBB6SE_add_source.mp4", "target": "videos/train_PVp8gPBB6SE_add_target.mp4" }, "remove": { "source": "videos/train_PVp8gPBB6SE_remove_source.mp4", "target": "videos/train_PVp8gPBB6SE_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with cries", "generated": "videos/train_PVp8gPBB6SE_withref_add.mp4", "metrics": { "l1_tgt": 28.7, "l1_src": 27.6, "std": 54.6, "gt_std": 55.0, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with cries", "generated": "videos/train_PVp8gPBB6SE_noref_add.mp4", "metrics": { "l1_tgt": 38.0, "l1_src": 38.1, "std": 59.3, "gt_std": 55.0, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without cries", "generated": "videos/train_PVp8gPBB6SE_withref_remove.mp4", "metrics": { "l1_tgt": 19.8, "l1_src": 32.3, "std": 49.2, "gt_std": 49.4, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without cries", "generated": "videos/train_PVp8gPBB6SE_noref_remove.mp4", "metrics": { "l1_tgt": 19.5, "l1_src": 32.5, "std": 50.3, "gt_std": 49.4, "audio": true } } ] }, { "video_id": "HQ5pWZscxlI_000034", "object": "electric guitar", "refs": { "add": { "source": "videos/train_HQ5pWZscxlI_000034_add_source.mp4", "target": "videos/train_HQ5pWZscxlI_000034_add_target.mp4" }, "remove": { "source": "videos/train_HQ5pWZscxlI_000034_remove_source.mp4", "target": "videos/train_HQ5pWZscxlI_000034_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with electric guitar", "generated": "videos/train_HQ5pWZscxlI_000034_withref_add.mp4", "metrics": { "l1_tgt": 28.0, "l1_src": 28.0, "std": 82.0, "gt_std": 77.4, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with electric guitar", "generated": "videos/train_HQ5pWZscxlI_000034_noref_add.mp4", "metrics": { "l1_tgt": 23.3, "l1_src": 22.7, "std": 79.7, "gt_std": 77.4, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without electric guitar", "generated": "videos/train_HQ5pWZscxlI_000034_withref_remove.mp4", "metrics": { "l1_tgt": 18.9, "l1_src": 21.2, "std": 72.7, "gt_std": 73.8, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without electric guitar", "generated": "videos/train_HQ5pWZscxlI_000034_noref_remove.mp4", "metrics": { "l1_tgt": 18.4, "l1_src": 18.3, "std": 74.2, "gt_std": 73.8, "audio": true } } ] }, { "video_id": "8WIo3L9z0_g", "object": "tool", "refs": { "add": { "source": "videos/train_8WIo3L9z0_g_add_source.mp4", "target": "videos/train_8WIo3L9z0_g_add_target.mp4" }, "remove": { "source": "videos/train_8WIo3L9z0_g_remove_source.mp4", "target": "videos/train_8WIo3L9z0_g_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with tool", "generated": "videos/train_8WIo3L9z0_g_withref_add.mp4", "metrics": { "l1_tgt": 34.6, "l1_src": 23.0, "std": 49.0, "gt_std": 55.7, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with tool", "generated": "videos/train_8WIo3L9z0_g_noref_add.mp4", "metrics": { "l1_tgt": 35.6, "l1_src": 25.9, "std": 49.6, "gt_std": 55.7, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without tool", "generated": "videos/train_8WIo3L9z0_g_withref_remove.mp4", "metrics": { "l1_tgt": 22.7, "l1_src": 32.4, "std": 43.0, "gt_std": 42.4, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without tool", "generated": "videos/train_8WIo3L9z0_g_noref_remove.mp4", "metrics": { "l1_tgt": 20.6, "l1_src": 30.7, "std": 41.7, "gt_std": 42.4, "audio": true } } ] }, { "video_id": "8UFTZixFVuc", "object": "piano", "refs": { "add": { "source": "videos/train_8UFTZixFVuc_add_source.mp4", "target": "videos/train_8UFTZixFVuc_add_target.mp4" }, "remove": { "source": "videos/train_8UFTZixFVuc_remove_source.mp4", "target": "videos/train_8UFTZixFVuc_remove_target.mp4" } }, "main": [ { "task": "add", "ref": "withref", "cfg": 1.0, "prompt": "a video with piano", "generated": "videos/train_8UFTZixFVuc_withref_add.mp4", "metrics": { "l1_tgt": 33.2, "l1_src": 18.4, "std": 50.7, "gt_std": 79.9, "audio": true } }, { "task": "add", "ref": "noref", "cfg": 1.0, "prompt": "a video with piano", "generated": "videos/train_8UFTZixFVuc_noref_add.mp4", "metrics": { "l1_tgt": 33.7, "l1_src": 14.8, "std": 46.9, "gt_std": 79.9, "audio": true } }, { "task": "remove", "ref": "withref", "cfg": 1.0, "prompt": "a video without piano", "generated": "videos/train_8UFTZixFVuc_withref_remove.mp4", "metrics": { "l1_tgt": 12.2, "l1_src": 38.4, "std": 39.9, "gt_std": 41.7, "audio": true } }, { "task": "remove", "ref": "noref", "cfg": 1.0, "prompt": "a video without piano", "generated": "videos/train_8UFTZixFVuc_noref_remove.mp4", "metrics": { "l1_tgt": 15.5, "l1_src": 43.4, "std": 32.8, "gt_std": 41.7, "audio": true } } ] } ] } ], "summary": { "val/add/withref": { "mean_gap": 9.7, "n_moved_right": 0, "n": 5 }, "val/add/noref": { "mean_gap": 8.2, "n_moved_right": 0, "n": 5 }, "val/remove/withref": { "mean_gap": -11.6, "n_moved_right": 5, "n": 5 }, "val/remove/noref": { "mean_gap": -6.3, "n_moved_right": 4, "n": 5 }, "train/add/withref": { "mean_gap": 7.3, "n_moved_right": 0, "n": 5 }, "train/add/noref": { "mean_gap": 7.7, "n_moved_right": 1, "n": 5 }, "train/remove/withref": { "mean_gap": -9.6, "n_moved_right": 4, "n": 5 }, "train/remove/noref": { "mean_gap": -11.2, "n_moved_right": 4, "n": 5 } } }