| { |
| "run_id": "text_both_base_0728", |
| "checkpoint": "epoch004-global_step17000", |
| "sampling_steps": 50, |
| "main_cfg": 1.0, |
| "splits": [ |
| { |
| "key": "val", |
| "label": "val.jsonl \u2014 held out", |
| "entries": [ |
| { |
| "video_id": "f1w8_fTIZZE_000036", |
| "object": "trumpets", |
| "refs": { |
| "add": { |
| "source": "videos/val_f1w8_fTIZZE_000036_add_source.mp4", |
| "target": "videos/val_f1w8_fTIZZE_000036_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/val_f1w8_fTIZZE_000036_remove_source.mp4", |
| "target": "videos/val_f1w8_fTIZZE_000036_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with trumpets", |
| "generated": "videos/val_f1w8_fTIZZE_000036_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 39.9, |
| "l1_src": 27.4, |
| "std": 75.6, |
| "gt_std": 74.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with trumpets", |
| "generated": "videos/val_f1w8_fTIZZE_000036_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 36.2, |
| "l1_src": 30.5, |
| "std": 75.7, |
| "gt_std": 74.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without trumpets", |
| "generated": "videos/val_f1w8_fTIZZE_000036_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 29.2, |
| "l1_src": 41.8, |
| "std": 62.4, |
| "gt_std": 61.8, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without trumpets", |
| "generated": "videos/val_f1w8_fTIZZE_000036_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 30.7, |
| "l1_src": 33.3, |
| "std": 64.4, |
| "gt_std": 61.8, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "gDfj4eERZ6Q_000200", |
| "object": "man", |
| "refs": { |
| "add": { |
| "source": "videos/val_gDfj4eERZ6Q_000200_add_source.mp4", |
| "target": "videos/val_gDfj4eERZ6Q_000200_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/val_gDfj4eERZ6Q_000200_remove_source.mp4", |
| "target": "videos/val_gDfj4eERZ6Q_000200_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with man", |
| "generated": "videos/val_gDfj4eERZ6Q_000200_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 22.4, |
| "l1_src": 18.4, |
| "std": 55.6, |
| "gt_std": 52.5, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with man", |
| "generated": "videos/val_gDfj4eERZ6Q_000200_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 22.9, |
| "l1_src": 18.7, |
| "std": 54.0, |
| "gt_std": 52.5, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without man", |
| "generated": "videos/val_gDfj4eERZ6Q_000200_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 26.4, |
| "l1_src": 30.9, |
| "std": 35.8, |
| "gt_std": 51.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without man", |
| "generated": "videos/val_gDfj4eERZ6Q_000200_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 14.3, |
| "l1_src": 16.1, |
| "std": 48.3, |
| "gt_std": 51.0, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "qWfbM-MKkgs", |
| "object": "train engine", |
| "refs": { |
| "add": { |
| "source": "videos/val_qWfbM-MKkgs_add_source.mp4", |
| "target": "videos/val_qWfbM-MKkgs_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/val_qWfbM-MKkgs_remove_source.mp4", |
| "target": "videos/val_qWfbM-MKkgs_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with train engine", |
| "generated": "videos/val_qWfbM-MKkgs_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 34.2, |
| "l1_src": 33.2, |
| "std": 68.7, |
| "gt_std": 70.8, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with train engine", |
| "generated": "videos/val_qWfbM-MKkgs_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 37.4, |
| "l1_src": 33.7, |
| "std": 66.5, |
| "gt_std": 70.8, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without train engine", |
| "generated": "videos/val_qWfbM-MKkgs_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 31.2, |
| "l1_src": 33.8, |
| "std": 64.8, |
| "gt_std": 59.6, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without train engine", |
| "generated": "videos/val_qWfbM-MKkgs_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 38.0, |
| "l1_src": 38.0, |
| "std": 66.0, |
| "gt_std": 59.6, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "20qZLse0acs_000030", |
| "object": "dog", |
| "refs": { |
| "add": { |
| "source": "videos/val_20qZLse0acs_000030_add_source.mp4", |
| "target": "videos/val_20qZLse0acs_000030_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/val_20qZLse0acs_000030_remove_source.mp4", |
| "target": "videos/val_20qZLse0acs_000030_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with dog", |
| "generated": "videos/val_20qZLse0acs_000030_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 36.4, |
| "l1_src": 18.4, |
| "std": 67.3, |
| "gt_std": 75.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with dog", |
| "generated": "videos/val_20qZLse0acs_000030_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 27.0, |
| "l1_src": 14.7, |
| "std": 64.2, |
| "gt_std": 75.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without dog", |
| "generated": "videos/val_20qZLse0acs_000030_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 10.5, |
| "l1_src": 26.9, |
| "std": 63.0, |
| "gt_std": 62.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without dog", |
| "generated": "videos/val_20qZLse0acs_000030_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 14.2, |
| "l1_src": 18.3, |
| "std": 66.3, |
| "gt_std": 62.7, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "vo3Vx7SxaNw_000030", |
| "object": "birds", |
| "refs": { |
| "add": { |
| "source": "videos/val_vo3Vx7SxaNw_000030_add_source.mp4", |
| "target": "videos/val_vo3Vx7SxaNw_000030_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/val_vo3Vx7SxaNw_000030_remove_source.mp4", |
| "target": "videos/val_vo3Vx7SxaNw_000030_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with birds", |
| "generated": "videos/val_vo3Vx7SxaNw_000030_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 48.6, |
| "l1_src": 35.6, |
| "std": 56.6, |
| "gt_std": 65.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with birds", |
| "generated": "videos/val_vo3Vx7SxaNw_000030_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 44.9, |
| "l1_src": 29.6, |
| "std": 49.6, |
| "gt_std": 65.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without birds", |
| "generated": "videos/val_vo3Vx7SxaNw_000030_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 24.0, |
| "l1_src": 45.9, |
| "std": 38.7, |
| "gt_std": 37.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without birds", |
| "generated": "videos/val_vo3Vx7SxaNw_000030_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 23.2, |
| "l1_src": 46.4, |
| "std": 39.3, |
| "gt_std": 37.0, |
| "audio": true |
| } |
| } |
| ] |
| } |
| ] |
| }, |
| { |
| "key": "train", |
| "label": "train.jsonl \u2014 seen in training", |
| "entries": [ |
| { |
| "video_id": "ITc_gC-oIUc_000064", |
| "object": "mandolin", |
| "refs": { |
| "add": { |
| "source": "videos/train_ITc_gC-oIUc_000064_add_source.mp4", |
| "target": "videos/train_ITc_gC-oIUc_000064_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/train_ITc_gC-oIUc_000064_remove_source.mp4", |
| "target": "videos/train_ITc_gC-oIUc_000064_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with mandolin", |
| "generated": "videos/train_ITc_gC-oIUc_000064_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 38.4, |
| "l1_src": 29.6, |
| "std": 76.2, |
| "gt_std": 76.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with mandolin", |
| "generated": "videos/train_ITc_gC-oIUc_000064_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 37.1, |
| "l1_src": 27.9, |
| "std": 80.4, |
| "gt_std": 76.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without mandolin", |
| "generated": "videos/train_ITc_gC-oIUc_000064_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 32.8, |
| "l1_src": 30.0, |
| "std": 71.7, |
| "gt_std": 68.5, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without mandolin", |
| "generated": "videos/train_ITc_gC-oIUc_000064_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 19.2, |
| "l1_src": 24.5, |
| "std": 72.5, |
| "gt_std": 68.5, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "PVp8gPBB6SE", |
| "object": "cries", |
| "refs": { |
| "add": { |
| "source": "videos/train_PVp8gPBB6SE_add_source.mp4", |
| "target": "videos/train_PVp8gPBB6SE_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/train_PVp8gPBB6SE_remove_source.mp4", |
| "target": "videos/train_PVp8gPBB6SE_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with cries", |
| "generated": "videos/train_PVp8gPBB6SE_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 28.7, |
| "l1_src": 27.6, |
| "std": 54.6, |
| "gt_std": 55.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with cries", |
| "generated": "videos/train_PVp8gPBB6SE_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 38.0, |
| "l1_src": 38.1, |
| "std": 59.3, |
| "gt_std": 55.0, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without cries", |
| "generated": "videos/train_PVp8gPBB6SE_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 19.8, |
| "l1_src": 32.3, |
| "std": 49.2, |
| "gt_std": 49.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without cries", |
| "generated": "videos/train_PVp8gPBB6SE_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 19.5, |
| "l1_src": 32.5, |
| "std": 50.3, |
| "gt_std": 49.4, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "HQ5pWZscxlI_000034", |
| "object": "electric guitar", |
| "refs": { |
| "add": { |
| "source": "videos/train_HQ5pWZscxlI_000034_add_source.mp4", |
| "target": "videos/train_HQ5pWZscxlI_000034_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/train_HQ5pWZscxlI_000034_remove_source.mp4", |
| "target": "videos/train_HQ5pWZscxlI_000034_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with electric guitar", |
| "generated": "videos/train_HQ5pWZscxlI_000034_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 28.0, |
| "l1_src": 28.0, |
| "std": 82.0, |
| "gt_std": 77.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with electric guitar", |
| "generated": "videos/train_HQ5pWZscxlI_000034_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 23.3, |
| "l1_src": 22.7, |
| "std": 79.7, |
| "gt_std": 77.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without electric guitar", |
| "generated": "videos/train_HQ5pWZscxlI_000034_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 18.9, |
| "l1_src": 21.2, |
| "std": 72.7, |
| "gt_std": 73.8, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without electric guitar", |
| "generated": "videos/train_HQ5pWZscxlI_000034_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 18.4, |
| "l1_src": 18.3, |
| "std": 74.2, |
| "gt_std": 73.8, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "8WIo3L9z0_g", |
| "object": "tool", |
| "refs": { |
| "add": { |
| "source": "videos/train_8WIo3L9z0_g_add_source.mp4", |
| "target": "videos/train_8WIo3L9z0_g_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/train_8WIo3L9z0_g_remove_source.mp4", |
| "target": "videos/train_8WIo3L9z0_g_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with tool", |
| "generated": "videos/train_8WIo3L9z0_g_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 34.6, |
| "l1_src": 23.0, |
| "std": 49.0, |
| "gt_std": 55.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with tool", |
| "generated": "videos/train_8WIo3L9z0_g_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 35.6, |
| "l1_src": 25.9, |
| "std": 49.6, |
| "gt_std": 55.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without tool", |
| "generated": "videos/train_8WIo3L9z0_g_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 22.7, |
| "l1_src": 32.4, |
| "std": 43.0, |
| "gt_std": 42.4, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without tool", |
| "generated": "videos/train_8WIo3L9z0_g_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 20.6, |
| "l1_src": 30.7, |
| "std": 41.7, |
| "gt_std": 42.4, |
| "audio": true |
| } |
| } |
| ] |
| }, |
| { |
| "video_id": "8UFTZixFVuc", |
| "object": "piano", |
| "refs": { |
| "add": { |
| "source": "videos/train_8UFTZixFVuc_add_source.mp4", |
| "target": "videos/train_8UFTZixFVuc_add_target.mp4" |
| }, |
| "remove": { |
| "source": "videos/train_8UFTZixFVuc_remove_source.mp4", |
| "target": "videos/train_8UFTZixFVuc_remove_target.mp4" |
| } |
| }, |
| "main": [ |
| { |
| "task": "add", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video with piano", |
| "generated": "videos/train_8UFTZixFVuc_withref_add.mp4", |
| "metrics": { |
| "l1_tgt": 33.2, |
| "l1_src": 18.4, |
| "std": 50.7, |
| "gt_std": 79.9, |
| "audio": true |
| } |
| }, |
| { |
| "task": "add", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video with piano", |
| "generated": "videos/train_8UFTZixFVuc_noref_add.mp4", |
| "metrics": { |
| "l1_tgt": 33.7, |
| "l1_src": 14.8, |
| "std": 46.9, |
| "gt_std": 79.9, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "withref", |
| "cfg": 1.0, |
| "prompt": "a video without piano", |
| "generated": "videos/train_8UFTZixFVuc_withref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 12.2, |
| "l1_src": 38.4, |
| "std": 39.9, |
| "gt_std": 41.7, |
| "audio": true |
| } |
| }, |
| { |
| "task": "remove", |
| "ref": "noref", |
| "cfg": 1.0, |
| "prompt": "a video without piano", |
| "generated": "videos/train_8UFTZixFVuc_noref_remove.mp4", |
| "metrics": { |
| "l1_tgt": 15.5, |
| "l1_src": 43.4, |
| "std": 32.8, |
| "gt_std": 41.7, |
| "audio": true |
| } |
| } |
| ] |
| } |
| ] |
| } |
| ], |
| "summary": { |
| "val/add/withref": { |
| "mean_gap": 9.7, |
| "n_moved_right": 0, |
| "n": 5 |
| }, |
| "val/add/noref": { |
| "mean_gap": 8.2, |
| "n_moved_right": 0, |
| "n": 5 |
| }, |
| "val/remove/withref": { |
| "mean_gap": -11.6, |
| "n_moved_right": 5, |
| "n": 5 |
| }, |
| "val/remove/noref": { |
| "mean_gap": -6.3, |
| "n_moved_right": 4, |
| "n": 5 |
| }, |
| "train/add/withref": { |
| "mean_gap": 7.3, |
| "n_moved_right": 0, |
| "n": 5 |
| }, |
| "train/add/noref": { |
| "mean_gap": 7.7, |
| "n_moved_right": 1, |
| "n": 5 |
| }, |
| "train/remove/withref": { |
| "mean_gap": -9.6, |
| "n_moved_right": 4, |
| "n": 5 |
| }, |
| "train/remove/noref": { |
| "mean_gap": -11.2, |
| "n_moved_right": 4, |
| "n": 5 |
| } |
| } |
| } |
