Add files using upload-large-folder tool
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +548 -0
- checkpoints/predictor_v4_1000_seed0/end_to_end.lock +0 -0
- checkpoints/predictor_v4_1000_seed0/end_to_end.log +355 -0
- checkpoints/predictor_v4_1000_seed0/launcher.log +381 -0
- checkpoints/predictor_v4_1000_seed0/swanlab/.gitignore +1 -0
- checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/debug/debug.log +27 -0
- checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/config.yaml +311 -0
- checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/requirements.txt +177 -0
- checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/swanlab-metadata.json +1 -0
- checkpoints/predictor_v4_1000_seed0/sync_watcher.log +3 -0
- checkpoints/predictor_v4_1000_seed0/train_config.json +144 -0
- checkpoints/predictor_v4_1000_seed0/train_log.jsonl +201 -0
- checkpoints/predictor_v4_1000_seed0/training_and_sync.lock +0 -0
- checkpoints/predictor_v4_1000_seed0/training_and_sync.log +421 -0
- checkpoints/predictor_v4_1000_seed0/training_and_sync_before_fresh_bs32_20260725.log +504 -0
- checkpoints/predictor_v4_1000_seed0/training_and_sync_stopped_step90.log +152 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/launcher.log +0 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/.gitignore +1 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/debug/debug.log +0 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/config.yaml +271 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/requirements.txt +177 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/swanlab-metadata.json +1 -0
- checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/train_config.json +131 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/launcher.log +0 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/.gitignore +1 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/debug/debug.log +0 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/config.yaml +271 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/requirements.txt +177 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/swanlab-metadata.json +1 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/train_config.json +131 -0
- checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/train_log.jsonl +0 -0
- checkpoints/predictor_v4_stage2_suite/run_suite.pid +1 -0
- checkpoints/predictor_v4_stage2_suite/tmux_session +1 -0
- checkpoints/predictor_v4_stage2_suite/wait_and_train.lock +0 -0
- checkpoints/predictor_v4_stage2_suite/wait_and_train.log +0 -0
- data/self_forcing_predictor_v4_1000_seed0/cases.jsonl +0 -0
- data/self_forcing_predictor_v4_1000_seed0/dataset_config.json +29 -0
- data/self_forcing_predictor_v4_1000_seed0/manifest_summary.json +12 -0
- data/self_forcing_predictor_v4_1000_seed0/selected_prompts.txt +0 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/0-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/1-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/11-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/12-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/15-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/16-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/18-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/20-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/26-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/30-0_ema.mp4 +3 -0
- videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/32-0_ema.mp4 +3 -0
.gitattributes
CHANGED
|
@@ -1016,3 +1016,551 @@ videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/98-0_ema.mp4 filter=lf
|
|
| 1016 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1017 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1018 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1016 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1017 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1018 |
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1019 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1020 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1021 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1022 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1023 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1024 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1025 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1026 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1027 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1028 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1029 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1030 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1031 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1032 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1033 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1034 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1035 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1036 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1037 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1038 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1039 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1040 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1041 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1042 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1043 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1044 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1045 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1046 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1047 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1048 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1049 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1050 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1051 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1052 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1053 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1054 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1055 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1056 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1057 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/57-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1058 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/51-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1059 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1060 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/53-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1061 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1062 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/33-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1063 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/79-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1064 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/90-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1065 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/8-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1066 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1067 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/58-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1068 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/10-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1069 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1070 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1071 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/2-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1072 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/64-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1073 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/24-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1074 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1075 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1076 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/99-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1077 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1078 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1079 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/23-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1080 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/47-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1081 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/17-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1082 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1083 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1084 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1085 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/59-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1086 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/77-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1087 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/40-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1088 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1089 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1090 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/31-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1091 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/72-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1092 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/62-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1093 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/21-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1094 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1095 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/60-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1096 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1097 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1098 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/34-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1099 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/70-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1100 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1101 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/22-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1102 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1103 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/65-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1104 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1105 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1106 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/13-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1107 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1108 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1109 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1110 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/82-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1111 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/6-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1112 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1113 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/54-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1114 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1115 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1116 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/67-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1117 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1118 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/68-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1119 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/4-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1120 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1121 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1122 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1123 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1124 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1125 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/88-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1126 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/85-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1127 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/83-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1128 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1129 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/7-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1130 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/93-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1131 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1132 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1133 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/27-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1134 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1135 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1136 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1137 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/36-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1138 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/87-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1139 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1140 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/14-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1141 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1142 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/61-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1143 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/41-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1144 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/98-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1145 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1146 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1147 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/35-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1148 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/55-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1149 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1150 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1151 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1152 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1153 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/50-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1154 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/29-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1155 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppf_seed0/92-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1156 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/92-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1157 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/60-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1158 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/6-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1159 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1160 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/82-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1161 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1162 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/35-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1163 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1164 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/31-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1165 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1166 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/34-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1167 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1168 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/36-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1169 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/62-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1170 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/27-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1171 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1172 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1173 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1174 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1175 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/99-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1176 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/57-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1177 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1178 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/54-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1179 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/58-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1180 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1181 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1182 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1183 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/4-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1184 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/85-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1185 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/51-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1186 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/8-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1187 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/88-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1188 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/24-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1189 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1190 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/22-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1191 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/61-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1192 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1193 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/2-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1194 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/21-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1195 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/98-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1196 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1197 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1198 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1199 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1200 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1201 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/79-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1202 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1203 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1204 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/53-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1205 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1206 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1207 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1208 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1209 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/70-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1210 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/64-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1211 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/68-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1212 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1213 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1214 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1215 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1216 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/10-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1217 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1218 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/14-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1219 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1220 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1221 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1222 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1223 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1224 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/7-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1225 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/90-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1226 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/40-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1227 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/59-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1228 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/65-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1229 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/23-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1230 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/87-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1231 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/41-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1232 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1233 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1234 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/77-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1235 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1236 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1237 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1238 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1239 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/13-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1240 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/47-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1241 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1242 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/17-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1243 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/50-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1244 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/93-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1245 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1246 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1247 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/72-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1248 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/67-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1249 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1250 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/33-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1251 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1252 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/83-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1253 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/29-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1254 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1255 |
+
videos/moviegenvideobench_extended_101_200_dmd_frrr_chunk0_ffff_seed0/55-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1256 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/65-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1257 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1258 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1259 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/22-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1260 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/13-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1261 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/58-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1262 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1263 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/53-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1264 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1265 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1266 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/70-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1267 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/33-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1268 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1269 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/41-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1270 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1271 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/68-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1272 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1273 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1274 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/57-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1275 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1276 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1277 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1278 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/35-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1279 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/62-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1280 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/99-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1281 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1282 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1283 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1284 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/36-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1285 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1286 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1287 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1288 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1289 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/79-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1290 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/82-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1291 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1292 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/88-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1293 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1294 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/21-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1295 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/67-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1296 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1297 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1298 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/54-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1299 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/64-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1300 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1301 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/60-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1302 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/8-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1303 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/90-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1304 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/77-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1305 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/98-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1306 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1307 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1308 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/14-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1309 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1310 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/50-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1311 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/24-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1312 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1313 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1314 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/87-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1315 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1316 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1317 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1318 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/61-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1319 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1320 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/2-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1321 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1322 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/59-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1323 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1324 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/31-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1325 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/55-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1326 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/72-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1327 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/6-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1328 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/85-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1329 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/47-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1330 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1331 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1332 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1333 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/17-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1334 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1335 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/23-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1336 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1337 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1338 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1339 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/92-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1340 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/93-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1341 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/83-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1342 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1343 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/51-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1344 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/40-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1345 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/4-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1346 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1347 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1348 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1349 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1350 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/34-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1351 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/7-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1352 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/27-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1353 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/10-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1354 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/29-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1355 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_offline_ffff_fppp_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1356 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1357 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1358 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/77-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1359 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/79-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1360 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1361 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1362 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/88-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1363 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/2-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1364 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/50-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1365 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1366 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1367 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1368 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1369 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/57-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1370 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1371 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/36-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1372 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1373 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/65-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1374 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/17-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1375 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/62-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1376 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/61-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1377 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/33-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1378 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/4-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1379 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1380 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/55-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1381 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1382 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/7-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1383 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/99-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1384 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/70-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1385 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/41-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1386 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1387 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/34-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1388 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/21-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1389 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/68-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1390 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/23-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1391 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/93-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1392 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/67-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1393 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/51-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1394 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1395 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/29-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1396 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1397 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1398 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1399 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1400 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1401 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1402 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1403 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/98-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1404 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1405 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/40-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1406 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1407 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/59-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1408 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1409 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/87-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1410 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1411 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/6-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1412 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1413 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1414 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1415 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1416 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1417 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/14-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1418 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1419 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/10-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1420 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1421 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/64-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1422 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1423 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/90-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1424 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/27-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1425 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1426 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1427 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/53-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1428 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1429 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1430 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/54-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1431 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/24-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1432 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/72-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1433 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1434 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1435 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/83-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1436 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1437 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/35-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1438 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/8-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1439 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/58-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1440 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/31-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1441 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/22-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1442 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1443 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1444 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/82-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1445 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/60-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1446 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/85-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1447 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1448 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/47-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1449 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1450 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/92-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1451 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/13-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1452 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1453 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1454 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1455 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage1_fppf_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1456 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1457 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/59-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1458 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/57-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1459 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1460 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/22-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1461 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/27-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1462 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/31-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1463 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/29-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1464 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/65-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1465 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/37-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1466 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/32-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1467 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/44-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1468 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/94-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1469 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/17-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1470 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/49-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1471 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/78-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1472 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/2-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1473 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/12-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1474 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/61-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1475 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/51-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1476 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/42-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1477 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/0-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1478 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/88-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1479 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/47-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1480 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/93-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1481 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1482 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/83-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1483 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/90-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1484 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/82-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1485 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/68-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1486 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1487 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/70-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1488 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/43-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1489 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/38-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1490 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/56-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1491 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/6-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1492 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/10-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1493 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/97-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1494 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/84-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1495 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1496 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/53-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1497 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/81-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1498 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/41-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1499 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/23-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1500 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/86-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1501 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/4-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1502 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/76-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1503 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/54-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1504 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/48-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1505 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/92-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1506 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/7-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1507 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/62-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1508 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/16-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1509 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/77-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1510 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1511 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/91-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1512 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/33-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1513 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/11-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1514 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1515 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/96-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1516 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/36-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1517 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1518 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/40-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1519 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1520 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/30-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1521 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/64-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1522 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/67-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1523 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/60-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1524 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/72-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1525 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/5-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1526 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/55-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1527 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/58-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1528 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/75-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1529 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/66-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1530 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/35-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1531 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/50-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1532 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/63-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1533 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/20-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1534 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/73-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1535 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/13-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1536 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/8-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1537 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/74-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1538 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/39-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1539 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/21-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1540 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/71-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1541 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/26-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1542 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/52-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1543 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/79-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1544 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/98-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1545 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/69-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1546 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/34-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1547 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/14-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1548 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/24-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1549 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/87-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1550 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1551 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1552 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/1-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1553 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/85-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1554 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/99-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1555 |
+
videos/moviegenvideobench_extended_first100_dmd_predictor_v4_stage2_onpolicy_fppp_seed0/18-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1556 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/25-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1557 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/19-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1558 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/89-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1559 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/15-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1560 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/46-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1561 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/95-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1562 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/9-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1563 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/80-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1564 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/28-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1565 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/45-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 1566 |
+
videos/moviegenvideobench_extended_101_200_dmd_stage2_offline_ffff_fppp_seed0/3-0_ema.mp4 filter=lfs diff=lfs merge=lfs -text
|
checkpoints/predictor_v4_1000_seed0/end_to_end.lock
ADDED
|
File without changes
|
checkpoints/predictor_v4_1000_seed0/end_to_end.log
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-24T10:19:17Z] GPU 尚忙，30s 后重试。
|
| 2 |
+
[2026-07-24T10:19:58Z] GPU 尚忙，30s 后重试。
|
| 3 |
+
[2026-07-24T10:20:28Z] GPU 尚忙，30s 后重试。
|
| 4 |
+
[2026-07-24T10:20:59Z] GPU 尚忙，30s 后重试。
|
| 5 |
+
[2026-07-24T10:21:29Z] GPU 尚忙，30s 后重试。
|
| 6 |
+
[2026-07-24T10:21:59Z] GPU 尚忙，30s 后重试。
|
| 7 |
+
[2026-07-24T10:22:29Z] GPU 尚忙，30s 后重试。
|
| 8 |
+
[2026-07-24T10:22:59Z] GPU 尚忙，30s 后重试。
|
| 9 |
+
[2026-07-24T10:23:29Z] GPU 尚忙，30s 后重试。
|
| 10 |
+
[2026-07-24T10:23:59Z] GPU 尚忙，30s 后重试。
|
| 11 |
+
[2026-07-24T10:24:29Z] GPU 尚忙，30s 后重试。
|
| 12 |
+
[2026-07-24T10:25:00Z] GPU 尚忙，30s 后重试。
|
| 13 |
+
[2026-07-24T10:25:30Z] GPU 尚忙，30s 后重试。
|
| 14 |
+
[2026-07-24T10:26:00Z] GPU 尚忙，30s 后重试。
|
| 15 |
+
[2026-07-24T10:26:30Z] GPU 尚忙，30s 后重试。
|
| 16 |
+
[2026-07-24T10:27:00Z] GPU 尚忙，30s 后重试。
|
| 17 |
+
[2026-07-24T10:27:30Z] GPU 尚忙，30s 后重试。
|
| 18 |
+
[2026-07-24T10:28:00Z] GPU 尚忙，30s 后重试。
|
| 19 |
+
[2026-07-24T10:28:31Z] GPU 尚忙，30s 后重试。
|
| 20 |
+
[2026-07-24T10:29:01Z] GPU 尚忙，30s 后重试。
|
| 21 |
+
[2026-07-24T10:29:31Z] GPU 尚忙，30s 后重试。
|
| 22 |
+
[2026-07-24T10:30:01Z] GPU 尚忙，30s 后重试。
|
| 23 |
+
[2026-07-24T10:30:31Z] GPU 尚忙，30s 后重试。
|
| 24 |
+
[2026-07-24T10:31:01Z] GPU 尚忙，30s 后重试。
|
| 25 |
+
[2026-07-24T10:31:31Z] GPU 尚忙，30s 后重试。
|
| 26 |
+
[2026-07-24T10:32:01Z] GPU 尚忙，30s 后重试。
|
| 27 |
+
[2026-07-24T10:32:32Z] GPU 尚忙，30s 后重试。
|
| 28 |
+
[2026-07-24T10:33:02Z] GPU 尚忙，30s 后重试。
|
| 29 |
+
[2026-07-24T10:33:32Z] GPU 尚忙，30s 后重试。
|
| 30 |
+
[2026-07-24T10:34:02Z] GPU 尚忙，30s 后重试。
|
| 31 |
+
[2026-07-24T10:34:32Z] GPU 尚忙，30s 后重试。
|
| 32 |
+
[2026-07-24T10:35:02Z] GPU 尚忙，30s 后重试。
|
| 33 |
+
[2026-07-24T10:35:32Z] GPU 尚忙，30s 后重试。
|
| 34 |
+
[2026-07-24T10:36:03Z] GPU 尚忙，30s 后重试。
|
| 35 |
+
[2026-07-24T10:36:33Z] GPU 尚忙，30s 后重试。
|
| 36 |
+
[2026-07-24T10:37:03Z] GPU 尚忙，30s 后重试。
|
| 37 |
+
[2026-07-24T10:37:33Z] GPU 尚忙，30s 后重试。
|
| 38 |
+
[2026-07-24T10:38:03Z] GPU 尚忙，30s 后重试。
|
| 39 |
+
[2026-07-24T10:38:33Z] GPU 尚忙，30s 后重试。
|
| 40 |
+
[2026-07-24T10:39:03Z] GPU 尚忙，30s 后重试。
|
| 41 |
+
[2026-07-24T10:39:33Z] GPU 尚忙，30s 后重试。
|
| 42 |
+
[2026-07-24T10:40:04Z] GPU 尚忙，30s 后重试。
|
| 43 |
+
[2026-07-24T10:40:34Z] GPU 尚忙，30s 后重试。
|
| 44 |
+
[2026-07-24T10:41:04Z] GPU 尚忙，30s 后重试。
|
| 45 |
+
[2026-07-24T10:41:34Z] GPU 尚忙，30s 后重试。
|
| 46 |
+
[2026-07-24T10:42:04Z] GPU 尚忙，30s 后重试。
|
| 47 |
+
[2026-07-24T10:42:34Z] GPU 尚忙，30s 后重试。
|
| 48 |
+
[2026-07-24T10:43:04Z] GPU 尚忙，30s 后重试。
|
| 49 |
+
[2026-07-24T10:43:35Z] GPU 尚忙，30s 后重试。
|
| 50 |
+
[2026-07-24T10:44:05Z] GPU 尚忙，30s 后重试。
|
| 51 |
+
[2026-07-24T10:44:35Z] GPU 尚忙，30s 后重试。
|
| 52 |
+
[2026-07-24T10:45:05Z] GPU 尚忙，30s 后重试。
|
| 53 |
+
[2026-07-24T10:45:35Z] GPU 尚忙，30s 后重试。
|
| 54 |
+
[2026-07-24T10:46:05Z] GPU 尚忙，30s 后重试。
|
| 55 |
+
[2026-07-24T10:46:35Z] GPU 尚忙，30s 后重试。
|
| 56 |
+
[2026-07-24T10:47:05Z] GPU 尚忙，30s 后重试。
|
| 57 |
+
[2026-07-24T10:47:36Z] GPU 尚忙，30s 后重试。
|
| 58 |
+
[2026-07-24T10:48:06Z] GPU 尚忙，30s 后重试。
|
| 59 |
+
[2026-07-24T10:48:36Z] GPU 尚忙，30s 后重试。
|
| 60 |
+
[2026-07-24T10:49:06Z] GPU 尚忙，30s 后重试。
|
| 61 |
+
[2026-07-24T10:49:36Z] GPU 尚忙，30s 后重试。
|
| 62 |
+
[2026-07-24T10:50:06Z] GPU 尚忙，30s 后重试。
|
| 63 |
+
[2026-07-24T10:50:36Z] GPU 尚忙，30s 后重试。
|
| 64 |
+
[2026-07-24T10:51:07Z] GPU 尚忙，30s 后重试。
|
| 65 |
+
[2026-07-24T10:51:37Z] GPU 尚忙，30s 后重试。
|
| 66 |
+
[2026-07-24T10:52:07Z] GPU 尚忙，30s 后重试。
|
| 67 |
+
[2026-07-24T10:52:37Z] GPU 尚忙，30s 后重试。
|
| 68 |
+
[2026-07-24T10:53:07Z] GPU 尚忙，30s 后重试。
|
| 69 |
+
[2026-07-24T10:53:37Z] GPU 尚忙，30s 后重试。
|
| 70 |
+
[2026-07-24T10:54:07Z] GPU 尚忙，30s 后重试。
|
| 71 |
+
[2026-07-24T10:54:37Z] GPU 尚忙，30s 后重试。
|
| 72 |
+
[2026-07-24T10:55:08Z] GPU 尚忙，30s 后重试。
|
| 73 |
+
[2026-07-24T10:55:38Z] GPU 尚忙，30s 后重试。
|
| 74 |
+
[2026-07-24T10:56:08Z] GPU 尚忙，30s 后重试。
|
| 75 |
+
[2026-07-24T10:56:38Z] GPU 尚忙，30s 后重试。
|
| 76 |
+
[2026-07-24T10:57:08Z] GPU 尚忙，30s 后重试。
|
| 77 |
+
[2026-07-24T10:57:38Z] GPU 尚忙，30s 后重试。
|
| 78 |
+
[2026-07-24T10:58:08Z] GPU 尚忙，30s 后重试。
|
| 79 |
+
[2026-07-24T10:58:39Z] GPU 尚忙，30s 后重试。
|
| 80 |
+
[2026-07-24T10:59:09Z] GPU 尚忙，30s 后重试。
|
| 81 |
+
[2026-07-24T10:59:39Z] GPU 尚忙，30s 后重试。
|
| 82 |
+
[2026-07-24T11:00:09Z] GPU 尚忙，30s 后重试。
|
| 83 |
+
[2026-07-24T11:00:39Z] GPU 尚忙，30s 后重试。
|
| 84 |
+
[2026-07-24T11:01:09Z] GPU 尚忙，30s 后重试。
|
| 85 |
+
[2026-07-24T11:01:39Z] GPU 尚忙，30s 后重试。
|
| 86 |
+
[2026-07-24T11:02:09Z] GPU 尚忙，30s 后重试。
|
| 87 |
+
[2026-07-24T11:02:40Z] GPU 尚忙，30s 后重试。
|
| 88 |
+
[2026-07-24T11:03:10Z] GPU 尚忙，30s 后重试。
|
| 89 |
+
[2026-07-24T11:03:40Z] GPU 尚忙，30s 后重试。
|
| 90 |
+
[2026-07-24T11:04:10Z] GPU 尚忙，30s 后重试。
|
| 91 |
+
[2026-07-24T11:04:40Z] GPU 尚忙，30s 后重试。
|
| 92 |
+
[2026-07-24T11:05:10Z] GPU 尚忙，30s 后重试。
|
| 93 |
+
[2026-07-24T11:05:40Z] GPU 尚忙，30s 后重试。
|
| 94 |
+
[2026-07-24T11:06:10Z] GPU 尚忙，30s 后重试。
|
| 95 |
+
[2026-07-24T11:06:41Z] GPU 尚忙，30s 后重试。
|
| 96 |
+
[2026-07-24T11:07:11Z] GPU 尚忙，30s 后重试。
|
| 97 |
+
[2026-07-24T11:07:41Z] GPU 尚忙，30s 后重试。
|
| 98 |
+
[2026-07-24T11:08:11Z] GPU 尚忙，30s 后重试。
|
| 99 |
+
[2026-07-24T11:08:41Z] GPU 尚忙，30s 后重试。
|
| 100 |
+
[2026-07-24T11:09:11Z] GPU 尚忙，30s 后重试。
|
| 101 |
+
[2026-07-24T11:09:41Z] GPU 尚忙，30s 后重试。
|
| 102 |
+
[2026-07-24T11:10:12Z] GPU 尚忙，30s 后重试。
|
| 103 |
+
[2026-07-24T11:10:42Z] GPU 尚忙，30s 后重试。
|
| 104 |
+
[2026-07-24T11:11:12Z] GPU 尚忙，30s 后重试。
|
| 105 |
+
[2026-07-24T11:11:42Z] GPU 尚忙，30s 后重试。
|
| 106 |
+
[2026-07-24T11:12:12Z] GPU 尚忙，30s 后重试。
|
| 107 |
+
[2026-07-24T11:12:42Z] GPU 尚忙，30s 后重试。
|
| 108 |
+
[2026-07-24T11:13:12Z] GPU 尚忙，30s 后重试。
|
| 109 |
+
[2026-07-24T11:13:42Z] GPU 尚忙，30s 后重试。
|
| 110 |
+
[2026-07-24T11:14:13Z] GPU 尚忙，30s 后重试。
|
| 111 |
+
[2026-07-24T11:14:43Z] GPU 尚忙，30s 后重试。
|
| 112 |
+
[2026-07-24T11:15:13Z] GPU 尚忙，30s 后重试。
|
| 113 |
+
[2026-07-24T11:15:43Z] GPU 尚忙，30s 后重试。
|
| 114 |
+
[2026-07-24T11:16:13Z] GPU 尚忙，30s 后重试。
|
| 115 |
+
[2026-07-24T11:16:43Z] GPU 尚忙，30s 后重试。
|
| 116 |
+
[2026-07-24T11:17:13Z] GPU 尚忙，30s 后重试。
|
| 117 |
+
[2026-07-24T11:17:43Z] GPU 尚忙，30s 后重试。
|
| 118 |
+
[2026-07-24T11:18:14Z] GPU 尚忙，30s 后重试。
|
| 119 |
+
[2026-07-24T11:18:44Z] GPU 尚忙，30s 后重试。
|
| 120 |
+
[2026-07-24T11:19:14Z] GPU 尚忙，30s 后重试。
|
| 121 |
+
[2026-07-24T11:19:44Z] GPU 尚忙，30s 后重试。
|
| 122 |
+
[2026-07-24T11:20:14Z] GPU 尚忙，30s 后重试。
|
| 123 |
+
[2026-07-24T11:20:44Z] GPU 尚忙，30s 后重试。
|
| 124 |
+
[2026-07-24T11:21:14Z] GPU 尚忙，30s 后重试。
|
| 125 |
+
[2026-07-24T11:21:44Z] GPU 尚忙，30s 后重试。
|
| 126 |
+
[2026-07-24T11:22:15Z] GPU 尚忙，30s 后重试。
|
| 127 |
+
[2026-07-24T11:22:45Z] GPU 尚忙，30s 后重试。
|
| 128 |
+
[2026-07-24T11:23:15Z] GPU 尚忙，30s 后重试。
|
| 129 |
+
[2026-07-24T11:23:45Z] GPU 尚忙，30s 后重试。
|
| 130 |
+
[2026-07-24T11:24:15Z] GPU 尚忙，30s 后重试。
|
| 131 |
+
[2026-07-24T11:24:45Z] GPU 尚忙，30s 后重试。
|
| 132 |
+
[2026-07-24T11:25:15Z] GPU 尚忙，30s 后重试。
|
| 133 |
+
[2026-07-24T11:25:46Z] GPU 尚忙，30s 后重试。
|
| 134 |
+
[2026-07-24T11:26:16Z] GPU 尚忙，30s 后重试。
|
| 135 |
+
[2026-07-24T11:26:46Z] GPU 尚忙，30s 后重试。
|
| 136 |
+
[2026-07-24T11:27:16Z] GPU 尚忙，30s 后重试。
|
| 137 |
+
[2026-07-24T11:27:46Z] GPU 尚忙，30s 后重试。
|
| 138 |
+
[2026-07-24T11:28:16Z] GPU 尚忙，30s 后重试。
|
| 139 |
+
[2026-07-24T11:28:46Z] GPU 尚忙，30s 后重试。
|
| 140 |
+
[2026-07-24T11:29:16Z] GPU 尚忙，30s 后重试。
|
| 141 |
+
[2026-07-24T11:29:47Z] GPU 尚忙，30s 后重试。
|
| 142 |
+
[2026-07-24T11:30:17Z] GPU 尚忙，30s 后重试。
|
| 143 |
+
[2026-07-24T11:30:47Z] GPU 尚忙，30s 后重试。
|
| 144 |
+
[2026-07-24T11:31:17Z] GPU 尚忙，30s 后重试。
|
| 145 |
+
[2026-07-24T11:31:47Z] GPU 尚忙，30s 后重试。
|
| 146 |
+
[2026-07-24T11:32:17Z] GPU 尚忙，30s 后重试。
|
| 147 |
+
[2026-07-24T11:32:47Z] GPU 尚忙，30s 后重试。
|
| 148 |
+
[2026-07-24T11:33:17Z] GPU 尚忙，30s 后重试。
|
| 149 |
+
[2026-07-24T11:33:48Z] GPU 尚忙，30s 后重试。
|
| 150 |
+
[2026-07-24T11:34:18Z] GPU 尚忙，30s 后重试。
|
| 151 |
+
[2026-07-24T11:34:48Z] GPU 尚忙，30s 后重试。
|
| 152 |
+
[2026-07-24T11:35:18Z] GPU 尚忙，30s 后重试。
|
| 153 |
+
[2026-07-24T11:35:48Z] GPU 尚忙，30s 后重试。
|
| 154 |
+
[2026-07-24T11:36:18Z] GPU 尚忙，30s 后重试。
|
| 155 |
+
[2026-07-24T11:36:48Z] GPU 尚忙，30s 后重试。
|
| 156 |
+
[2026-07-24T11:37:19Z] GPU 尚忙，30s 后重试。
|
| 157 |
+
[2026-07-24T11:37:49Z] GPU 尚忙，30s 后重试。
|
| 158 |
+
[2026-07-24T11:38:19Z] GPU 尚忙，30s 后重试。
|
| 159 |
+
[2026-07-24T11:38:49Z] GPU 尚忙，30s 后重试。
|
| 160 |
+
[2026-07-24T11:39:19Z] GPU 尚忙，30s 后重试。
|
| 161 |
+
[2026-07-24T11:39:49Z] GPU 尚忙，30s 后重试。
|
| 162 |
+
[2026-07-24T11:40:19Z] GPU 尚忙，30s 后重试。
|
| 163 |
+
[2026-07-24T11:40:50Z] GPU 尚忙，30s 后重试。
|
| 164 |
+
[2026-07-24T11:41:20Z] GPU 尚忙，30s 后重试。
|
| 165 |
+
[2026-07-24T11:41:50Z] GPU 尚忙，30s 后重试。
|
| 166 |
+
[2026-07-24T11:42:20Z] GPU 尚忙，30s 后重试。
|
| 167 |
+
[2026-07-24T11:42:50Z] GPU 尚忙，30s 后重试。
|
| 168 |
+
[2026-07-24T11:43:20Z] GPU 尚忙，30s 后重试。
|
| 169 |
+
[2026-07-24T11:43:50Z] GPU 尚忙，30s 后重试。
|
| 170 |
+
[2026-07-24T11:44:20Z] GPU 尚忙，30s 后重试。
|
| 171 |
+
[2026-07-24T11:44:51Z] GPU 尚忙，30s 后重试。
|
| 172 |
+
[2026-07-24T11:45:21Z] GPU 尚忙，30s 后重试。
|
| 173 |
+
[2026-07-24T11:45:51Z] GPU 尚忙，30s 后重试。
|
| 174 |
+
[2026-07-24T11:46:21Z] GPU 尚忙，30s 后重试。
|
| 175 |
+
[2026-07-24T11:46:51Z] GPU 尚忙，30s 后重试。
|
| 176 |
+
[2026-07-24T11:47:21Z] GPU 尚忙，30s 后重试。
|
| 177 |
+
[2026-07-24T11:47:51Z] GPU 尚忙，30s 后重试。
|
| 178 |
+
[2026-07-24T11:48:22Z] GPU 尚忙，30s 后重试。
|
| 179 |
+
[2026-07-24T11:48:52Z] GPU 尚忙，30s 后重试。
|
| 180 |
+
[2026-07-24T11:49:22Z] GPU 尚忙，30s 后重试。
|
| 181 |
+
[2026-07-24T11:49:52Z] GPU 尚忙，30s 后重试。
|
| 182 |
+
[2026-07-24T11:50:22Z] GPU 尚忙，30s 后重试。
|
| 183 |
+
[2026-07-24T11:50:52Z] GPU 尚忙，30s 后重试。
|
| 184 |
+
[2026-07-24T11:51:22Z] GPU 尚忙，30s 后重试。
|
| 185 |
+
[2026-07-24T11:51:52Z] GPU 尚忙，30s 后重试。
|
| 186 |
+
[2026-07-24T11:52:23Z] GPU 尚忙，30s 后重试。
|
| 187 |
+
[2026-07-24T11:52:53Z] GPU 尚忙，30s 后重试。
|
| 188 |
+
[2026-07-24T11:53:23Z] GPU 尚忙，30s 后重试。
|
| 189 |
+
[2026-07-24T11:53:53Z] GPU 尚忙，30s 后重试。
|
| 190 |
+
[2026-07-24T11:54:23Z] GPU 尚忙，30s 后重试。
|
| 191 |
+
[2026-07-24T11:54:53Z] GPU 尚忙，30s 后重试。
|
| 192 |
+
[2026-07-24T11:55:23Z] GPU 尚忙，30s 后重试。
|
| 193 |
+
[2026-07-24T11:55:54Z] GPU 尚忙，30s 后重试。
|
| 194 |
+
[2026-07-24T11:56:24Z] GPU 尚忙，30s 后重试。
|
| 195 |
+
[2026-07-24T11:56:54Z] GPU 尚忙，30s 后重试。
|
| 196 |
+
[2026-07-24T11:57:24Z] GPU 尚忙，30s 后重试。
|
| 197 |
+
[2026-07-24T11:57:54Z] GPU 尚忙，30s 后重试。
|
| 198 |
+
[2026-07-24T11:58:24Z] GPU 尚忙，30s 后重试。
|
| 199 |
+
[2026-07-24T11:58:54Z] GPU 尚忙，30s 后重试。
|
| 200 |
+
[2026-07-24T11:59:24Z] GPU 尚忙，30s 后重试。
|
| 201 |
+
[2026-07-24T11:59:55Z] GPU 尚忙，30s 后重试。
|
| 202 |
+
[2026-07-24T12:00:25Z] GPU 尚忙，30s 后重试。
|
| 203 |
+
[2026-07-24T12:00:55Z] GPU 尚忙，30s 后重试。
|
| 204 |
+
[2026-07-24T12:01:25Z] GPU 尚忙，30s 后重试。
|
| 205 |
+
[2026-07-24T12:01:55Z] GPU 尚忙，30s 后重试。
|
| 206 |
+
[2026-07-24T12:02:25Z] GPU 尚忙，30s 后重试。
|
| 207 |
+
[2026-07-24T12:02:55Z] GPU 尚忙，30s 后重试。
|
| 208 |
+
[2026-07-24T12:03:25Z] GPU 尚忙，30s 后重试。
|
| 209 |
+
[2026-07-24T12:03:56Z] GPU 尚忙，30s 后重试。
|
| 210 |
+
[2026-07-24T12:04:26Z] GPU 尚忙，30s 后重试。
|
| 211 |
+
[2026-07-24T12:04:56Z] GPU 尚忙，30s 后重试。
|
| 212 |
+
[2026-07-24T12:05:26Z] GPU 尚忙，30s 后重试。
|
| 213 |
+
[2026-07-24T12:05:56Z] GPU 尚忙，30s 后重试。
|
| 214 |
+
[2026-07-24T12:06:26Z] GPU 尚忙，30s 后重试。
|
| 215 |
+
[2026-07-24T12:06:56Z] GPU 尚忙，30s 后重试。
|
| 216 |
+
[2026-07-24T12:07:26Z] GPU 尚忙，30s 后重试。
|
| 217 |
+
[2026-07-24T12:07:57Z] GPU 尚忙，30s 后重试。
|
| 218 |
+
[2026-07-24T12:08:27Z] GPU 尚忙，30s 后重试。
|
| 219 |
+
[2026-07-24T12:08:57Z] GPU 尚忙，30s 后重试。
|
| 220 |
+
[2026-07-24T12:09:27Z] GPU 尚忙，30s 后重试。
|
| 221 |
+
[2026-07-24T12:09:57Z] GPU 尚忙，30s 后重试。
|
| 222 |
+
[2026-07-24T12:10:27Z] GPU 尚忙，30s 后重试。
|
| 223 |
+
[2026-07-24T12:10:57Z] GPU 尚忙，30s 后重试。
|
| 224 |
+
[2026-07-24T12:11:27Z] GPU 尚忙，30s 后重试。
|
| 225 |
+
[2026-07-24T12:11:58Z] GPU 尚忙，30s 后重试。
|
| 226 |
+
[2026-07-24T12:12:28Z] GPU 尚忙，30s 后重试。
|
| 227 |
+
[2026-07-24T12:12:58Z] GPU 尚忙，30s 后重试。
|
| 228 |
+
[2026-07-24T12:13:28Z] GPU 尚忙，30s 后重试。
|
| 229 |
+
[2026-07-24T12:13:58Z] GPU 尚忙，30s 后重试。
|
| 230 |
+
[2026-07-24T12:14:28Z] GPU 尚忙，30s 后重试。
|
| 231 |
+
[2026-07-24T12:14:58Z] GPU 尚忙，30s 后重试。
|
| 232 |
+
[2026-07-24T12:15:29Z] GPU 尚忙，30s 后重试。
|
| 233 |
+
[2026-07-24T12:15:59Z] GPU 尚忙，30s 后重试。
|
| 234 |
+
[2026-07-24T12:16:29Z] GPU 尚忙，30s 后重试。
|
| 235 |
+
[2026-07-24T12:16:59Z] GPU 尚忙，30s 后重试。
|
| 236 |
+
[2026-07-24T12:17:29Z] GPU 尚忙，30s 后重试。
|
| 237 |
+
[2026-07-24T12:17:59Z] GPU 尚忙，30s 后重试。
|
| 238 |
+
[2026-07-24T12:18:29Z] GPU 尚忙，30s 后重试。
|
| 239 |
+
[2026-07-24T12:18:59Z] GPU 尚忙，30s 后重试。
|
| 240 |
+
[2026-07-24T12:19:30Z] GPU 尚忙，30s 后重试。
|
| 241 |
+
[2026-07-24T12:20:00Z] GPU 尚忙，30s 后重试。
|
| 242 |
+
[2026-07-24T12:20:30Z] GPU 尚忙，30s 后重试。
|
| 243 |
+
[2026-07-24T12:21:00Z] GPU 尚忙，30s 后重试。
|
| 244 |
+
[2026-07-24T12:21:30Z] GPU 尚忙，30s 后重试。
|
| 245 |
+
[2026-07-24T12:22:00Z] GPU 尚忙，30s 后重试。
|
| 246 |
+
[2026-07-24T12:22:30Z] GPU 尚忙，30s 后重试。
|
| 247 |
+
[2026-07-24T12:23:00Z] GPU 尚忙，30s 后重试。
|
| 248 |
+
[2026-07-24T12:23:31Z] GPU 尚忙，30s 后重试。
|
| 249 |
+
[2026-07-24T12:24:01Z] GPU 尚忙，30s 后重试。
|
| 250 |
+
[2026-07-24T12:24:31Z] GPU 尚忙，30s 后重试。
|
| 251 |
+
[2026-07-24T12:25:01Z] GPU 尚忙，30s 后重试。
|
| 252 |
+
[2026-07-24T12:25:31Z] GPU 尚忙，30s 后重试。
|
| 253 |
+
[2026-07-24T12:26:01Z] GPU 尚忙，30s 后重试。
|
| 254 |
+
[2026-07-24T12:26:31Z] GPU 尚忙，30s 后重试。
|
| 255 |
+
[2026-07-24T12:27:02Z] GPU 尚忙，30s 后重试。
|
| 256 |
+
[2026-07-24T12:27:32Z] GPU 尚忙，30s 后重试。
|
| 257 |
+
[2026-07-24T12:28:02Z] GPU 尚忙，30s 后重试。
|
| 258 |
+
[2026-07-24T12:28:32Z] GPU 尚忙，30s 后重试。
|
| 259 |
+
[2026-07-24T12:29:02Z] GPU 尚忙，30s 后重试。
|
| 260 |
+
[2026-07-24T12:29:32Z] GPU 尚忙，30s 后重试。
|
| 261 |
+
[2026-07-24T12:30:02Z] GPU 尚忙，30s 后重试。
|
| 262 |
+
[2026-07-24T12:30:32Z] GPU 尚忙，30s 后重试。
|
| 263 |
+
[2026-07-24T12:31:03Z] GPU 尚忙，30s 后重试。
|
| 264 |
+
[2026-07-24T12:31:33Z] GPU 尚忙，30s 后重试。
|
| 265 |
+
[2026-07-24T12:32:03Z] GPU 尚忙，30s 后重试。
|
| 266 |
+
[2026-07-24T12:32:33Z] GPU 尚忙，30s 后重试。
|
| 267 |
+
[2026-07-24T12:33:03Z] GPU 尚忙，30s 后重试。
|
| 268 |
+
[2026-07-24T12:33:33Z] GPU 尚忙，30s 后重试。
|
| 269 |
+
[2026-07-24T12:34:03Z] GPU 尚忙，30s 后重试。
|
| 270 |
+
[2026-07-24T12:34:34Z] GPU 尚忙，30s 后重试。
|
| 271 |
+
[2026-07-24T12:35:04Z] GPU 尚忙，30s 后重试。
|
| 272 |
+
[2026-07-24T12:35:34Z] GPU 尚忙，30s 后重试。
|
| 273 |
+
[2026-07-24T12:36:04Z] GPU 尚忙，30s 后重试。
|
| 274 |
+
[2026-07-24T12:36:34Z] GPU 尚忙，30s 后重试。
|
| 275 |
+
[2026-07-24T12:37:05Z] GPU 尚忙，30s 后重试。
|
| 276 |
+
[2026-07-24T12:37:35Z] GPU 尚忙，30s 后重试。
|
| 277 |
+
[2026-07-24T12:38:05Z] GPU 尚忙，30s 后重试。
|
| 278 |
+
[2026-07-24T12:38:35Z] GPU 尚忙，30s 后重试。
|
| 279 |
+
[2026-07-24T12:39:05Z] GPU 尚忙，30s 后重试。
|
| 280 |
+
[2026-07-24T12:39:35Z] GPU 尚忙，30s 后重试。
|
| 281 |
+
[2026-07-24T12:40:05Z] GPU 尚忙，30s 后重试。
|
| 282 |
+
[2026-07-24T12:40:36Z] GPU 尚忙，30s 后重试。
|
| 283 |
+
[2026-07-24T12:41:06Z] GPU 尚忙，30s 后重试。
|
| 284 |
+
[2026-07-24T12:41:36Z] GPU 尚忙，30s 后重试。
|
| 285 |
+
[2026-07-24T12:42:06Z] GPU 尚忙，30s 后重试。
|
| 286 |
+
[2026-07-24T12:42:36Z] GPU 尚忙，30s 后重试。
|
| 287 |
+
[2026-07-24T12:43:06Z] GPU 尚忙，30s 后重试。
|
| 288 |
+
[2026-07-24T12:43:36Z] GPU 尚忙，30s 后重试。
|
| 289 |
+
[2026-07-24T12:44:06Z] GPU 尚忙，30s 后重试。
|
| 290 |
+
[2026-07-24T12:44:37Z] GPU 尚忙，30s 后重试。
|
| 291 |
+
[2026-07-24T12:45:07Z] GPU 尚忙，30s 后重试。
|
| 292 |
+
[2026-07-24T12:45:37Z] GPU 尚忙，30s 后重试。
|
| 293 |
+
[2026-07-24T12:46:07Z] GPU 尚忙，30s 后重试。
|
| 294 |
+
[2026-07-24T12:46:37Z] GPU 尚忙，30s 后重试。
|
| 295 |
+
[2026-07-24T12:47:07Z] GPU 尚忙，30s 后重试。
|
| 296 |
+
[2026-07-24T12:47:37Z] GPU 尚忙，30s 后重试。
|
| 297 |
+
[2026-07-24T12:48:07Z] GPU 尚忙，30s 后重试。
|
| 298 |
+
[2026-07-24T12:48:38Z] 开始单 case GPU 冒烟测试。
|
| 299 |
+
Selected 1 prompts into /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0; excluded exact matches against the first 100 non-empty MovieGen prompts.
|
| 300 |
+
[worker 0] assigned 1 cases using case_id % 1
|
| 301 |
+
KV inference with 3 frames per block
|
| 302 |
+
[run] worker=0 case=0 (1/1)
|
| 303 |
+
[saved] worker=0 case=0 chunk=0/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_00.safetensors
|
| 304 |
+
[saved] worker=0 case=0 chunk=1/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_01.safetensors
|
| 305 |
+
[saved] worker=0 case=0 chunk=2/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_02.safetensors
|
| 306 |
+
[saved] worker=0 case=0 chunk=3/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_03.safetensors
|
| 307 |
+
[saved] worker=0 case=0 chunk=4/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_04.safetensors
|
| 308 |
+
[saved] worker=0 case=0 chunk=5/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_05.safetensors
|
| 309 |
+
[saved] worker=0 case=0 chunk=6/6 path=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_smoke_seed0/steps/case_000000/chunk_06.safetensors
|
| 310 |
+
Merged 7 chunk records from 1 cases; train_manifest has 6 chunks / 18 adjacent-step pairs.
|
| 311 |
+
Validated 7 manifest records / 6 training chunks; fully loaded 7 tensor records.
|
| 312 |
+
[2026-07-24T12:50:41Z] 单 case GPU 冒烟测试通过。
|
| 313 |
+
[2026-07-24T12:50:41Z] 开始 1000 prompts / 8 GPU 离线数据构建。
|
| 314 |
+
Predictor v4 离线数据构建：
|
| 315 |
+
Dataset: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 316 |
+
GPUs: 0,1,2,3,4,5,6,7
|
| 317 |
+
Workers: 8
|
| 318 |
+
Teacher: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt [generator_ema]
|
| 319 |
+
Schedule: F-F-F-F
|
| 320 |
+
Seed: 每个 case 重置为 0
|
| 321 |
+
Blocks: 0 1 28 29
|
| 322 |
+
KV check: 0
|
| 323 |
+
已启动 worker 0，GPU=0，PID=1298435
|
| 324 |
+
已启动 worker 1，GPU=1，PID=1298436
|
| 325 |
+
已启动 worker 2，GPU=2，PID=1298437
|
| 326 |
+
已启动 worker 3，GPU=3，PID=1298438
|
| 327 |
+
已启动 worker 4，GPU=4，PID=1298439
|
| 328 |
+
已启动 worker 5，GPU=5，PID=1298440
|
| 329 |
+
已启动 worker 6，GPU=6，PID=1298441
|
| 330 |
+
已启动 worker 7，GPU=7，PID=1298442
|
| 331 |
+
Merged 7000 chunk records from 1000 cases; train_manifest has 6000 chunks / 18000 adjacent-step pairs.
|
| 332 |
+
Validated 7000 manifest records / 6000 training chunks; fully loaded 16 tensor records.
|
| 333 |
+
Predictor v4 离线数据构建和验证完成：/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 334 |
+
Predictor v4 离线数据构建：
|
| 335 |
+
Dataset: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 336 |
+
GPUs: 0,1,2,3,4,5,6,7
|
| 337 |
+
Workers: 8
|
| 338 |
+
Teacher: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt [generator_ema]
|
| 339 |
+
Schedule: F-F-F-F
|
| 340 |
+
Seed: 每个 case 重置为 0
|
| 341 |
+
Blocks: 0 1 28 29
|
| 342 |
+
KV check: 0
|
| 343 |
+
已启动 worker 0，GPU=0，PID=1315722
|
| 344 |
+
已启动 worker 1，GPU=1，PID=1315723
|
| 345 |
+
已启动 worker 2，GPU=2，PID=1315724
|
| 346 |
+
已启动 worker 3，GPU=3，PID=1315725
|
| 347 |
+
已启动 worker 4，GPU=4，PID=1315726
|
| 348 |
+
已启动 worker 5，GPU=5，PID=1315727
|
| 349 |
+
已启动 worker 6，GPU=6，PID=1315728
|
| 350 |
+
已启动 worker 7，GPU=7，PID=1315729
|
| 351 |
+
Merged 7000 chunk records from 1000 cases; train_manifest has 6000 chunks / 18000 adjacent-step pairs.
|
| 352 |
+
Validated 7000 manifest records / 6000 training chunks; fully loaded 16 tensor records.
|
| 353 |
+
Predictor v4 离线数据构建和验证完成：/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 354 |
+
[2026-07-24T13:40:18Z] 数据构建完成，开始 8 GPU Predictor-v4 训练。
|
| 355 |
+
scripts/run_predictor_v4_end_to_end.sh: line 100: NVME_TRAIN_ROOT: unbound variable
|
checkpoints/predictor_v4_1000_seed0/launcher.log
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
WARNING:__main__:
|
| 2 |
+
*****************************************
|
| 3 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 4 |
+
*****************************************
|
| 5 |
+
[rank0]:[W725 14:41:19.028815623 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 6 |
+
[rank4]:[W725 14:41:20.741616993 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 7 |
+
[rank1]:[W725 14:41:20.764551054 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 8 |
+
[rank3]:[W725 14:41:20.778364499 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 9 |
+
[rank5]:[W725 14:41:20.781348410 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 10 |
+
[rank6]:[W725 14:41:20.783009518 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 11 |
+
[rank7]:[W725 14:41:20.784113397 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 12 |
+
[rank2]:[W725 14:41:20.784201989 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 13 |
+
swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use
|
| 14 |
+
`swanlab login --relogin` to force relogin
|
| 15 |
+
swanlab: Tracking run with swanlab version 0.9.0
|
| 16 |
+
swanlab: 💾 Run data saved at
|
| 17 |
+
/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/
|
| 18 |
+
run-20260725_144159-4kxa9v7b
|
| 19 |
+
swanlab: 📁 View project at
|
| 20 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 21 |
+
swanlab: 🚀 View run pre...8-prompts1000-seed0-bs32x8-from-scratch at
|
| 22 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/4kxa9v7b
|
| 23 |
+
{"swanlab_run_id": "4kxa9v7b"}
|
| 24 |
+
{
|
| 25 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 26 |
+
"prompt_count": 1000,
|
| 27 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 28 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 29 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 30 |
+
"max_records": null,
|
| 31 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 32 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 33 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 34 |
+
"strict_teacher_load": true,
|
| 35 |
+
"source_block_ids": [
|
| 36 |
+
1,
|
| 37 |
+
28
|
| 38 |
+
],
|
| 39 |
+
"timestep_shift": 5.0,
|
| 40 |
+
"local_attn_size": -1,
|
| 41 |
+
"sink_size": 0,
|
| 42 |
+
"latent_height": 60,
|
| 43 |
+
"latent_width": 104,
|
| 44 |
+
"max_steps": 2000,
|
| 45 |
+
"batch_size": 32,
|
| 46 |
+
"gradient_accumulation_steps": 1,
|
| 47 |
+
"num_workers": 0,
|
| 48 |
+
"pin_memory": false,
|
| 49 |
+
"persistent_workers": false,
|
| 50 |
+
"fusion_lr": 0.0001,
|
| 51 |
+
"blocks_lr": 1e-05,
|
| 52 |
+
"beta1": 0.9,
|
| 53 |
+
"beta2": 0.95,
|
| 54 |
+
"weight_decay": 0.01,
|
| 55 |
+
"scheduler_warmup_steps": 100,
|
| 56 |
+
"block_warmup_steps": 100,
|
| 57 |
+
"hidden_loss_weight": 0.1,
|
| 58 |
+
"flow_loss_weight": 1.0,
|
| 59 |
+
"grad_clip": 1.0,
|
| 60 |
+
"fp32_trainable_params": true,
|
| 61 |
+
"gradient_checkpointing": false,
|
| 62 |
+
"find_unused_parameters": false,
|
| 63 |
+
"log_every": 10,
|
| 64 |
+
"save_every": 100,
|
| 65 |
+
"keep_snapshots": 20,
|
| 66 |
+
"resume": null,
|
| 67 |
+
"use_wandb": false,
|
| 68 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 69 |
+
"wandb_entity": null,
|
| 70 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 71 |
+
"use_swanlab": true,
|
| 72 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 73 |
+
"swanlab_workspace": null,
|
| 74 |
+
"swanlab_experiment": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs32x8-from-scratch",
|
| 75 |
+
"swanlab_description": "Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training",
|
| 76 |
+
"swanlab_tags": [
|
| 77 |
+
"predictor-v4",
|
| 78 |
+
"self-forcing",
|
| 79 |
+
"wan2.1-1.3b",
|
| 80 |
+
"offline-prefeature",
|
| 81 |
+
"blocks-1-28",
|
| 82 |
+
"8gpu",
|
| 83 |
+
"seed0"
|
| 84 |
+
],
|
| 85 |
+
"swanlab_mode": "cloud",
|
| 86 |
+
"supervision_pairs": [
|
| 87 |
+
[
|
| 88 |
+
0,
|
| 89 |
+
1
|
| 90 |
+
],
|
| 91 |
+
[
|
| 92 |
+
1,
|
| 93 |
+
2
|
| 94 |
+
],
|
| 95 |
+
[
|
| 96 |
+
2,
|
| 97 |
+
3
|
| 98 |
+
]
|
| 99 |
+
],
|
| 100 |
+
"manifest_records": 6000,
|
| 101 |
+
"dataset_pairs": 18000,
|
| 102 |
+
"steps_per_epoch_per_rank": 66,
|
| 103 |
+
"world_size": 8,
|
| 104 |
+
"global_batch_size": 256,
|
| 105 |
+
"trainable_parameters": 104684032,
|
| 106 |
+
"parameter_groups": {
|
| 107 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 108 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 109 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 110 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 111 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 112 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 113 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 114 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 115 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 116 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 117 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 118 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 119 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 120 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 121 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 122 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 123 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 124 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 125 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 126 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 127 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 128 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 129 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 130 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 131 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 132 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 133 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 134 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 135 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 136 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 137 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 138 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 139 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 140 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 141 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 142 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 143 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 144 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 145 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 146 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 147 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 148 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 149 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 150 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 151 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 152 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 153 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 154 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 155 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 156 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 157 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 158 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 159 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 160 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 161 |
+
"residual_out.weight": "fusion_residual",
|
| 162 |
+
"residual_out.bias": "fusion_residual"
|
| 163 |
+
},
|
| 164 |
+
"activation_dtype": "bfloat16",
|
| 165 |
+
"trainable_parameter_dtype": "float32",
|
| 166 |
+
"swanlab_run_id": "4kxa9v7b"
|
| 167 |
+
}
|
| 168 |
+
{"avg_micro_time_s": 7.782912929877057, "epoch": 0, "flow_mse": 1.728674739599228, "global_step": 1, "grad_norm_max": 1.224261999130249, "hidden_mse": 0.34973957017064095, "loss": 1.7636487036943436, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 59.944087982177734}
|
| 169 |
+
{"avg_micro_time_s": 1.852714978063078, "epoch": 0, "flow_mse": 1.7325291699833338, "global_step": 10, "grad_norm_max": 1.5914783477783203, "hidden_mse": 0.36041637013355887, "loss": 1.7685708122120962, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 68.46298551559448}
|
| 170 |
+
{"avg_micro_time_s": 1.846399891503097, "epoch": 0, "flow_mse": 1.699481675028801, "global_step": 20, "grad_norm_max": 1.9605422019958496, "hidden_mse": 0.38648081049323085, "loss": 1.7381297543644907, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 171 |
+
{"avg_micro_time_s": 2.076498315717618, "epoch": 0, "flow_mse": 1.5910302087664605, "global_step": 30, "grad_norm_max": 1.6393431425094604, "hidden_mse": 0.6486873880028725, "loss": 1.6558989509940147, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 172 |
+
{"avg_micro_time_s": 1.8261107150639875, "epoch": 0, "flow_mse": 1.4074327528476716, "global_step": 40, "grad_norm_max": 1.6300629377365112, "hidden_mse": 1.5101009473204614, "loss": 1.5584428489208222, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 173 |
+
{"avg_micro_time_s": 1.7557343540451258, "epoch": 0, "flow_mse": 1.3375679016113282, "global_step": 50, "grad_norm_max": 1.179707407951355, "hidden_mse": 1.7423540219664575, "loss": 1.5118033051490785, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 68.46298551559448}
|
| 174 |
+
{"avg_micro_time_s": 1.9954431485108217, "epoch": 0, "flow_mse": 1.288495972752571, "global_step": 60, "grad_norm_max": 0.9528158903121948, "hidden_mse": 1.6803042054176331, "loss": 1.4565263986587524, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 175 |
+
{"avg_micro_time_s": 2.073848069201631, "epoch": 1, "flow_mse": 1.2329863071441651, "global_step": 70, "grad_norm_max": 2.1396095752716064, "hidden_mse": 1.468385949730873, "loss": 1.3798249036073686, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 176 |
+
{"avg_micro_time_s": 1.9025867714255584, "epoch": 1, "flow_mse": 1.1653944417834283, "global_step": 80, "grad_norm_max": 3.283475875854492, "hidden_mse": 1.3008124545216562, "loss": 1.2954756870865822, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 177 |
+
{"avg_micro_time_s": 1.8846276455253248, "epoch": 1, "flow_mse": 1.0928176268935204, "global_step": 90, "grad_norm_max": 2.976677179336548, "hidden_mse": 1.1887190774083138, "loss": 1.211689528822899, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 178 |
+
{"avg_micro_time_s": 2.0524098566529574, "epoch": 1, "flow_mse": 1.0091206416487695, "global_step": 100, "grad_norm_max": 2.0720155239105225, "hidden_mse": 1.1053067579865457, "loss": 1.119651308655739, "lr_blocks": 1.0000000000000001e-07, "lr_fusion": 0.0001, "peak_memory_gib": 68.46298551559448}
|
| 179 |
+
{"avg_micro_time_s": 1.8157492470301806, "epoch": 1, "flow_mse": 0.9281245060265064, "global_step": 110, "grad_norm_max": 3.5477075576782227, "hidden_mse": 1.0355910658836365, "loss": 1.031683611869812, "lr_blocks": 1.1e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 68.46298551559448}
|
| 180 |
+
{"avg_micro_time_s": 1.7559494119937882, "epoch": 1, "flow_mse": 0.8380454987287522, "global_step": 120, "grad_norm_max": 3.569852828979492, "hidden_mse": 1.0150623381137849, "loss": 0.9395517356693746, "lr_blocks": 2.1000000000000002e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 68.46298551559448}
|
| 181 |
+
{"avg_micro_time_s": 1.9936403229308781, "epoch": 1, "flow_mse": 0.7462111793458462, "global_step": 130, "grad_norm_max": 2.9960243701934814, "hidden_mse": 0.9869170427322388, "loss": 0.8449028812348843, "lr_blocks": 3.1000000000000004e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 68.46298551559448}
|
| 182 |
+
{"avg_micro_time_s": 1.8761029311470339, "epoch": 2, "flow_mse": 0.6552131280303002, "global_step": 140, "grad_norm_max": 3.033362627029419, "hidden_mse": 0.9761701941490174, "loss": 0.7528301522135735, "lr_blocks": 4.1e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 68.46298551559448}
|
| 183 |
+
{"avg_micro_time_s": 1.6636652560366203, "epoch": 2, "flow_mse": 0.5641707576811313, "global_step": 150, "grad_norm_max": 3.3895790576934814, "hidden_mse": 0.9558152981102467, "loss": 0.6597522847354412, "lr_blocks": 5.1e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 68.46298551559448}
|
| 184 |
+
{"avg_micro_time_s": 1.8629546527619825, "epoch": 2, "flow_mse": 0.49576797895133495, "global_step": 160, "grad_norm_max": 4.9101152420043945, "hidden_mse": 0.9376181185245515, "loss": 0.5895297914743424, "lr_blocks": 6.1e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 68.46298551559448}
|
| 185 |
+
{"avg_micro_time_s": 1.865570171797299, "epoch": 2, "flow_mse": 0.44515644647181035, "global_step": 170, "grad_norm_max": 3.2292189598083496, "hidden_mse": 0.9383152291178704, "loss": 0.5389879673719407, "lr_blocks": 7.100000000000001e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 68.46298551559448}
|
| 186 |
+
{"avg_micro_time_s": 2.0450479036619074, "epoch": 2, "flow_mse": 0.38247467912733557, "global_step": 180, "grad_norm_max": 2.7824766635894775, "hidden_mse": 0.9554134920239449, "loss": 0.47801603078842164, "lr_blocks": 8.1e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 68.46298551559448}
|
| 187 |
+
{"avg_micro_time_s": 2.016261681757169, "epoch": 2, "flow_mse": 0.3393749196082354, "global_step": 190, "grad_norm_max": 2.3986711502075195, "hidden_mse": 0.9583184562623501, "loss": 0.4352067667990923, "lr_blocks": 9.100000000000001e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 68.46298551559448}
|
| 188 |
+
{"avg_micro_time_s": 1.955125979850709, "epoch": 3, "flow_mse": 0.2995503291487694, "global_step": 200, "grad_norm_max": 2.834411144256592, "hidden_mse": 0.9311102323234082, "loss": 0.39266135394573215, "lr_blocks": 1e-05, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 68.46298551559448}
|
| 189 |
+
{"avg_micro_time_s": 1.9450525438660407, "epoch": 3, "flow_mse": 0.26215458251535895, "global_step": 210, "grad_norm_max": 2.5127832889556885, "hidden_mse": 0.9175012007355691, "loss": 0.35390470623970033, "lr_blocks": 9.999238475781957e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 68.46298551559448}
|
| 190 |
+
{"avg_micro_time_s": 1.794988799588464, "epoch": 3, "flow_mse": 0.24451670069247486, "global_step": 220, "grad_norm_max": 3.86234712600708, "hidden_mse": 0.8481401547789574, "loss": 0.32933071814477444, "lr_blocks": 9.99695413509548e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 68.46298551559448}
|
| 191 |
+
{"avg_micro_time_s": 1.821383213158697, "epoch": 3, "flow_mse": 0.21927626319229604, "global_step": 230, "grad_norm_max": 2.894408702850342, "hidden_mse": 0.7950924828648568, "loss": 0.2987855151295662, "lr_blocks": 9.993147673772869e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 68.46298551559448}
|
| 192 |
+
{"avg_micro_time_s": 1.7578791950494634, "epoch": 3, "flow_mse": 0.21038782224059105, "global_step": 240, "grad_norm_max": 4.587767124176025, "hidden_mse": 0.7239097714424134, "loss": 0.2827788006514311, "lr_blocks": 9.987820251299121e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 68.46298551559448}
|
| 193 |
+
{"avg_micro_time_s": 1.9880367784600823, "epoch": 3, "flow_mse": 0.18143612183630467, "global_step": 250, "grad_norm_max": 2.522402048110962, "hidden_mse": 0.6568384729325771, "loss": 0.24711996987462045, "lr_blocks": 9.980973490458728e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 68.46298551559448}
|
| 194 |
+
{"avg_micro_time_s": 2.1574960700992962, "epoch": 3, "flow_mse": 0.17069688346236944, "global_step": 260, "grad_norm_max": 2.6453685760498047, "hidden_mse": 0.5946614742279053, "loss": 0.23016303274780514, "lr_blocks": 9.972609476841368e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 68.46298551559448}
|
| 195 |
+
{"avg_micro_time_s": 1.7216765362391016, "epoch": 4, "flow_mse": 0.16511701364070178, "global_step": 270, "grad_norm_max": 4.101485252380371, "hidden_mse": 0.5324683826416731, "loss": 0.21836385373026135, "lr_blocks": 9.962730758206612e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 68.46298551559448}
|
| 196 |
+
{"avg_micro_time_s": 2.036255760942004, "epoch": 4, "flow_mse": 0.15243881475180387, "global_step": 280, "grad_norm_max": 3.777620792388916, "hidden_mse": 0.49607212990522387, "loss": 0.20204602926969528, "lr_blocks": 9.951340343707852e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 68.46298551559448}
|
| 197 |
+
2026-07-25 15:13:30.859 | ERROR |
|
| 198 |
+
swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST
|
| 199 |
+
https://api.swanlab.cn/api/house/metrics -> 522 (5106ms) trace:unknown | [ERR]
|
| 200 |
+
code=unknown code message=unknown error
|
| 201 |
+
2026-07-25 15:13:33.726 | WARNING |
|
| 202 |
+
swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due
|
| 203 |
+
to network or server issues, retrying automatically.
|
| 204 |
+
swanlab: Upload recovered, resuming normally.
|
| 205 |
+
{"avg_micro_time_s": 2.026448777728365, "epoch": 4, "flow_mse": 0.13280713185667992, "global_step": 290, "grad_norm_max": 2.864901542663574, "hidden_mse": 0.45720995105803014, "loss": 0.1785281287506223, "lr_blocks": 9.938441702975689e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 68.46298551559448}
|
| 206 |
+
{"avg_micro_time_s": 1.7816235546153623, "epoch": 4, "flow_mse": 0.1316212204284966, "global_step": 300, "grad_norm_max": 3.557399272918701, "hidden_mse": 0.4236030507832766, "loss": 0.17398152723908425, "lr_blocks": 9.924038765061042e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 68.46298551559448}
|
| 207 |
+
{"avg_micro_time_s": 1.7509075272682821, "epoch": 4, "flow_mse": 0.12680668467655778, "global_step": 310, "grad_norm_max": 2.9080326557159424, "hidden_mse": 0.3970677562057972, "loss": 0.16651346255093813, "lr_blocks": 9.908135917238321e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 68.46298551559448}
|
| 208 |
+
{"avg_micro_time_s": 1.94211900010705, "epoch": 4, "flow_mse": 0.1265168657526374, "global_step": 320, "grad_norm_max": 3.0853631496429443, "hidden_mse": 0.3776782535016537, "loss": 0.16428469233214857, "lr_blocks": 9.890738003669029e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 68.46298551559448}
|
| 209 |
+
{"avg_micro_time_s": 1.9201412342023105, "epoch": 4, "flow_mse": 0.11589663326740265, "global_step": 330, "grad_norm_max": 3.0462610721588135, "hidden_mse": 0.3677100505679846, "loss": 0.1526676405221224, "lr_blocks": 9.871850323926178e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 68.46298551559448}
|
| 210 |
+
{"avg_micro_time_s": 1.8802110167060164, "epoch": 5, "flow_mse": 0.10986252082511783, "global_step": 340, "grad_norm_max": 2.5844221115112305, "hidden_mse": 0.3474765542894602, "loss": 0.14461017716675997, "lr_blocks": 9.851478631379982e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 68.46298551559448}
|
| 211 |
+
{"avg_micro_time_s": 1.970779871637933, "epoch": 5, "flow_mse": 0.10572945503517986, "global_step": 350, "grad_norm_max": 2.6753783226013184, "hidden_mse": 0.33289663605391984, "loss": 0.13901912048459053, "lr_blocks": 9.829629131445342e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 68.46298551559448}
|
| 212 |
+
{"avg_micro_time_s": 1.8511785630384112, "epoch": 5, "flow_mse": 0.09914825893938542, "global_step": 360, "grad_norm_max": 2.757209539413452, "hidden_mse": 0.3192139830440283, "loss": 0.13106965916231275, "lr_blocks": 9.806308479691595e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 68.46298551559448}
|
| 213 |
+
{"avg_micro_time_s": 1.9199401149497135, "epoch": 5, "flow_mse": 0.0996528490446508, "global_step": 370, "grad_norm_max": 2.9345085620880127, "hidden_mse": 0.3086499966681004, "loss": 0.1305178500711918, "lr_blocks": 9.781523779815178e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 68.46298551559448}
|
| 214 |
+
{"avg_micro_time_s": 1.7542048408882693, "epoch": 5, "flow_mse": 0.11152176447212697, "global_step": 380, "grad_norm_max": 3.106746196746826, "hidden_mse": 0.30627303943037987, "loss": 0.1421490689739585, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 68.46298551559448}
|
| 215 |
+
{"avg_micro_time_s": 1.9700474178520382, "epoch": 5, "flow_mse": 0.10044981110841036, "global_step": 390, "grad_norm_max": 2.560649871826172, "hidden_mse": 0.2952960841357708, "loss": 0.1299794196151197, "lr_blocks": 9.727592877996585e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 68.46298551559448}
|
| 216 |
+
{"avg_micro_time_s": 1.6663271824698314, "epoch": 6, "flow_mse": 0.09705260461196304, "global_step": 400, "grad_norm_max": 2.532449722290039, "hidden_mse": 0.29162609074264767, "loss": 0.1262152137234807, "lr_blocks": 9.698463103929542e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 68.46298551559448}
|
| 217 |
+
{"avg_micro_time_s": 1.904068420661497, "epoch": 6, "flow_mse": 0.09287562295794488, "global_step": 410, "grad_norm_max": 2.16715145111084, "hidden_mse": 0.28314792811870576, "loss": 0.121190416906029, "lr_blocks": 9.667902132486009e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 68.46298551559448}
|
| 218 |
+
{"avg_micro_time_s": 1.744727985771897, "epoch": 6, "flow_mse": 0.09119896441698075, "global_step": 420, "grad_norm_max": 2.299543619155884, "hidden_mse": 0.275063331797719, "loss": 0.1187052981927991, "lr_blocks": 9.635919272833938e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 68.46298551559448}
|
| 219 |
+
{"avg_micro_time_s": 1.7956668802158675, "epoch": 6, "flow_mse": 0.08759928597137333, "global_step": 430, "grad_norm_max": 2.317425489425659, "hidden_mse": 0.2642714731395245, "loss": 0.11402643444016576, "lr_blocks": 9.602524267262202e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 68.46298551559448}
|
| 220 |
+
{"avg_micro_time_s": 1.9157415576424683, "epoch": 6, "flow_mse": 0.08734634332358837, "global_step": 440, "grad_norm_max": 2.760629415512085, "hidden_mse": 0.25739369336515666, "loss": 0.11308571305125953, "lr_blocks": 9.567727288213005e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 68.46298551559448}
|
| 221 |
+
{"avg_micro_time_s": 1.8787613131338732, "epoch": 6, "flow_mse": 0.08993547437712551, "global_step": 450, "grad_norm_max": 3.014104127883911, "hidden_mse": 0.25076083578169345, "loss": 0.11501155886799097, "lr_blocks": 9.531538935183252e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 68.46298551559448}
|
| 222 |
+
{"avg_micro_time_s": 1.8499236255913276, "epoch": 6, "flow_mse": 0.0833338218741119, "global_step": 460, "grad_norm_max": 2.0357143878936768, "hidden_mse": 0.24471132159233094, "loss": 0.10780495461076499, "lr_blocks": 9.493970231495836e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 68.46298551559448}
|
| 223 |
+
{"avg_micro_time_s": 1.8277902282425202, "epoch": 7, "flow_mse": 0.08892108229920269, "global_step": 470, "grad_norm_max": 2.870374917984009, "hidden_mse": 0.24110151771456004, "loss": 0.11303123468533159, "lr_blocks": 9.45503262094184e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 68.46298551559448}
|
| 224 |
+
{"avg_micro_time_s": 1.8054274757610984, "epoch": 7, "flow_mse": 0.08921196823939681, "global_step": 480, "grad_norm_max": 2.4050748348236084, "hidden_mse": 0.24109693542122843, "loss": 0.113321662414819, "lr_blocks": 9.414737964294636e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 225 |
+
{"avg_micro_time_s": 1.969536912314652, "epoch": 7, "flow_mse": 0.08481648499146105, "global_step": 490, "grad_norm_max": 4.0341668128967285, "hidden_mse": 0.23553607985377312, "loss": 0.10837009344249965, "lr_blocks": 9.37309853569698e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 68.46298551559448}
|
| 226 |
+
{"avg_micro_time_s": 1.7191957566945348, "epoch": 7, "flow_mse": 0.08171517765149475, "global_step": 500, "grad_norm_max": 2.7221522331237793, "hidden_mse": 0.22536225505173207, "loss": 0.10425140429288149, "lr_blocks": 9.330127018922195e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 68.46298551559448}
|
| 227 |
+
{"avg_micro_time_s": 1.8753900891068043, "epoch": 7, "flow_mse": 0.08210981907323003, "global_step": 510, "grad_norm_max": 1.9840606451034546, "hidden_mse": 0.22030116170644762, "loss": 0.10413993652909995, "lr_blocks": 9.285836503510562e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 68.46298551559448}
|
| 228 |
+
{"avg_micro_time_s": 1.9803101174184121, "epoch": 7, "flow_mse": 0.08212232678197325, "global_step": 520, "grad_norm_max": 2.06720232963562, "hidden_mse": 0.21767251621931794, "loss": 0.10388957895338535, "lr_blocks": 9.24024048078213e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 68.46298551559448}
|
| 229 |
+
{"avg_micro_time_s": 1.7816847848371253, "epoch": 8, "flow_mse": 0.07924596662633121, "global_step": 530, "grad_norm_max": 2.3816592693328857, "hidden_mse": 0.21423672661185267, "loss": 0.10066964011639357, "lr_blocks": 9.193352839727122e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 68.46298551559448}
|
| 230 |
+
{"avg_micro_time_s": 1.862245862412965, "epoch": 8, "flow_mse": 0.07841471419669688, "global_step": 540, "grad_norm_max": 2.3286426067352295, "hidden_mse": 0.2106100643053651, "loss": 0.09947572173550726, "lr_blocks": 9.145187862775208e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 68.46298551559448}
|
| 231 |
+
{"avg_micro_time_s": 2.0419705960011925, "epoch": 8, "flow_mse": 0.07392775514163077, "global_step": 550, "grad_norm_max": 1.3650901317596436, "hidden_mse": 0.2003183823078871, "loss": 0.09395959414541721, "lr_blocks": 9.09576022144496e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 68.46298551559448}
|
| 232 |
+
{"avg_micro_time_s": 1.6736296482136823, "epoch": 8, "flow_mse": 0.07821477656252683, "global_step": 560, "grad_norm_max": 2.5323493480682373, "hidden_mse": 0.19945396259427073, "loss": 0.09816017346456647, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 68.46298551559448}
|
| 233 |
+
{"avg_micro_time_s": 2.003066141484305, "epoch": 8, "flow_mse": 0.08001610483042897, "global_step": 570, "grad_norm_max": 2.0001752376556396, "hidden_mse": 0.19836203493177892, "loss": 0.09985230900347233, "lr_blocks": 8.993177550236464e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 68.46298551559448}
|
| 234 |
+
{"avg_micro_time_s": 1.9908761504004362, "epoch": 8, "flow_mse": 0.07824829332530499, "global_step": 580, "grad_norm_max": 2.1289918422698975, "hidden_mse": 0.19441615901887419, "loss": 0.09768990930169821, "lr_blocks": 8.94005376803361e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 68.46298551559448}
|
| 235 |
+
{"avg_micro_time_s": 1.722032713617955, "epoch": 8, "flow_mse": 0.07608461757190527, "global_step": 590, "grad_norm_max": 2.0547220706939697, "hidden_mse": 0.19045738708227875, "loss": 0.09513035686686636, "lr_blocks": 8.885729807284855e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 68.46298551559448}
|
| 236 |
+
{"avg_micro_time_s": 1.5744449160105434, "epoch": 9, "flow_mse": 0.07611198453232647, "global_step": 600, "grad_norm_max": 1.646276831626892, "hidden_mse": 0.1878601059317589, "loss": 0.09489799588918686, "lr_blocks": 8.83022221559489e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 68.46298551559448}
|
| 237 |
+
{"avg_micro_time_s": 1.9617489955402563, "epoch": 9, "flow_mse": 0.07234360408037901, "global_step": 610, "grad_norm_max": 2.1270275115966797, "hidden_mse": 0.18017836939543486, "loss": 0.09036144157871605, "lr_blocks": 8.773547901113862e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 68.46298551559448}
|
| 238 |
+
{"avg_micro_time_s": 1.899630580819212, "epoch": 9, "flow_mse": 0.07547258702106774, "global_step": 620, "grad_norm_max": 2.535357713699341, "hidden_mse": 0.17847571074962618, "loss": 0.09332015877589583, "lr_blocks": 8.715724127386971e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 68.46298551559448}
|
| 239 |
+
{"avg_micro_time_s": 1.9009491362783593, "epoch": 9, "flow_mse": 0.07615209170617164, "global_step": 630, "grad_norm_max": 2.885998487472534, "hidden_mse": 0.17413493599742652, "loss": 0.09356558537110687, "lr_blocks": 8.656768508095853e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 68.46298551559448}
|
| 240 |
+
{"avg_micro_time_s": 2.033017601488973, "epoch": 9, "flow_mse": 0.07332893046550452, "global_step": 640, "grad_norm_max": 1.3615809679031372, "hidden_mse": 0.17425470147281885, "loss": 0.09075440168380738, "lr_blocks": 8.596699001693257e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 68.46298551559448}
|
| 241 |
+
{"avg_micro_time_s": 1.6646606730137137, "epoch": 9, "flow_mse": 0.07165623712353408, "global_step": 650, "grad_norm_max": 2.018874406814575, "hidden_mse": 0.17005239613354206, "loss": 0.08866147724911572, "lr_blocks": 8.535533905932739e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 68.46298551559448}
|
| 242 |
+
{"avg_micro_time_s": 2.01382767639152, "epoch": 9, "flow_mse": 0.07549470965750515, "global_step": 660, "grad_norm_max": 1.9090533256530762, "hidden_mse": 0.1704067310318351, "loss": 0.0925353833474219, "lr_blocks": 8.473291852294986e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 68.46298551559448}
|
| 243 |
+
{"avg_micro_time_s": 2.044479904185573, "epoch": 10, "flow_mse": 0.07384380707517267, "global_step": 670, "grad_norm_max": 1.6318471431732178, "hidden_mse": 0.16633677743375302, "loss": 0.09047748530283571, "lr_blocks": 8.409991800312493e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 68.46298551559448}
|
| 244 |
+
{"avg_micro_time_s": 1.994344039564021, "epoch": 10, "flow_mse": 0.07081504138186574, "global_step": 680, "grad_norm_max": 2.8086788654327393, "hidden_mse": 0.16047870106995107, "loss": 0.08686291193589568, "lr_blocks": 8.345653031794292e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 68.46298551559448}
|
| 245 |
+
{"avg_micro_time_s": 1.7797691442654469, "epoch": 10, "flow_mse": 0.07098656282760203, "global_step": 690, "grad_norm_max": 1.6218206882476807, "hidden_mse": 0.15934279719367625, "loss": 0.086920843180269, "lr_blocks": 8.280295144952537e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 68.46298551559448}
|
| 246 |
+
{"avg_micro_time_s": 1.8434631714044372, "epoch": 10, "flow_mse": 0.07692104973830283, "global_step": 700, "grad_norm_max": 1.9783955812454224, "hidden_mse": 0.16042201202362777, "loss": 0.0929632511921227, "lr_blocks": 8.213938048432697e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 68.46298551559448}
|
| 247 |
+
{"avg_micro_time_s": 1.8933369300531924, "epoch": 10, "flow_mse": 0.06859853272326291, "global_step": 710, "grad_norm_max": 1.2764592170715332, "hidden_mse": 0.15258000474423172, "loss": 0.08385653356090189, "lr_blocks": 8.146601955249187e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 68.46298551559448}
|
| 248 |
+
{"avg_micro_time_s": 1.863453762442805, "epoch": 10, "flow_mse": 0.0677291950210929, "global_step": 720, "grad_norm_max": 2.756103992462158, "hidden_mse": 0.15030786208808422, "loss": 0.08275998155586423, "lr_blocks": 8.078307376628292e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 68.46298551559448}
|
| 249 |
+
{"avg_micro_time_s": 1.5886019194789698, "epoch": 11, "flow_mse": 0.06930862842127682, "global_step": 730, "grad_norm_max": 2.032028913497925, "hidden_mse": 0.14793342733755707, "loss": 0.08410197151824833, "lr_blocks": 8.009075115760243e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 68.46298551559448}
|
| 250 |
+
{"avg_micro_time_s": 1.7562155689141947, "epoch": 11, "flow_mse": 0.0670456328894943, "global_step": 740, "grad_norm_max": 1.5815403461456299, "hidden_mse": 0.14644069485366346, "loss": 0.08168970262631775, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 68.46298551559448}
|
| 251 |
+
{"avg_micro_time_s": 1.9935554676223548, "epoch": 11, "flow_mse": 0.06759789921343327, "global_step": 750, "grad_norm_max": 1.5702470541000366, "hidden_mse": 0.1456328984349966, "loss": 0.08216118961572648, "lr_blocks": 7.86788218175523e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 68.46298551559448}
|
| 252 |
+
{"avg_micro_time_s": 1.7392390850000083, "epoch": 11, "flow_mse": 0.0679298588540405, "global_step": 760, "grad_norm_max": 1.7798969745635986, "hidden_mse": 0.145010856539011, "loss": 0.0824309445451945, "lr_blocks": 7.795964517353734e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 68.46298551559448}
|
| 253 |
+
{"avg_micro_time_s": 1.883487955624878, "epoch": 11, "flow_mse": 0.06899964096955956, "global_step": 770, "grad_norm_max": 2.0383424758911133, "hidden_mse": 0.14355416363105178, "loss": 0.08335505775175989, "lr_blocks": 7.723195175075136e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 68.46298551559448}
|
| 254 |
+
{"avg_micro_time_s": 1.902795129803417, "epoch": 11, "flow_mse": 0.06839722348377109, "global_step": 780, "grad_norm_max": 1.341465950012207, "hidden_mse": 0.14199498109519482, "loss": 0.08259672177955509, "lr_blocks": 7.649596321166024e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 68.46298551559448}
|
| 255 |
+
{"avg_micro_time_s": 1.899291706715303, "epoch": 11, "flow_mse": 0.06829775697551668, "global_step": 790, "grad_norm_max": 1.6974772214889526, "hidden_mse": 0.14167611030861735, "loss": 0.08246536822989584, "lr_blocks": 7.575190374550272e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 68.46298551559448}
|
| 256 |
+
{"avg_micro_time_s": 1.9210877921635985, "epoch": 12, "flow_mse": 0.06724138637073339, "global_step": 800, "grad_norm_max": 1.6519050598144531, "hidden_mse": 0.13736099740490318, "loss": 0.08097748612053693, "lr_blocks": 7.500000000000001e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 68.46298551559448}
|
| 257 |
+
{"avg_micro_time_s": 1.839930806370103, "epoch": 12, "flow_mse": 0.06486202948726714, "global_step": 810, "grad_norm_max": 1.9736137390136719, "hidden_mse": 0.13331869533285498, "loss": 0.07819389933720232, "lr_blocks": 7.424048101231687e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 68.46298551559448}
|
| 258 |
+
{"avg_micro_time_s": 1.7203275968320668, "epoch": 12, "flow_mse": 0.06997489500790835, "global_step": 820, "grad_norm_max": 1.8273866176605225, "hidden_mse": 0.1383979432284832, "loss": 0.0838146897032857, "lr_blocks": 7.347357813929455e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 68.46298551559448}
|
| 259 |
+
{"avg_micro_time_s": 1.8308625894540456, "epoch": 12, "flow_mse": 0.06103807487525046, "global_step": 830, "grad_norm_max": 1.24065363407135, "hidden_mse": 0.12885685674846173, "loss": 0.07392376055940986, "lr_blocks": 7.269952498697734e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 68.46298551559448}
|
| 260 |
+
{"avg_micro_time_s": 1.951346460581408, "epoch": 12, "flow_mse": 0.0672075655311346, "global_step": 840, "grad_norm_max": 1.4299647808074951, "hidden_mse": 0.1341157573275268, "loss": 0.08061914164572954, "lr_blocks": 7.191855733945388e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 68.46298551559448}
|
| 261 |
+
{"avg_micro_time_s": 1.9852554764482193, "epoch": 12, "flow_mse": 0.06508104703389109, "global_step": 850, "grad_norm_max": 1.3047852516174316, "hidden_mse": 0.13088483419269323, "loss": 0.07816953058354557, "lr_blocks": 7.113091308703498e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 68.46298551559448}
|
| 262 |
+
{"avg_micro_time_s": 1.8463190050082632, "epoch": 13, "flow_mse": 0.06611472489312291, "global_step": 860, "grad_norm_max": 1.3949657678604126, "hidden_mse": 0.13323927726596593, "loss": 0.07943865223787726, "lr_blocks": 7.033683215379002e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 263 |
+
{"avg_micro_time_s": 1.8538904657267268, "epoch": 13, "flow_mse": 0.0621907118242234, "global_step": 870, "grad_norm_max": 1.0381734371185303, "hidden_mse": 0.1276529028080404, "loss": 0.07495600231923163, "lr_blocks": 6.953655642446368e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 68.46298551559448}
|
| 264 |
+
{"avg_micro_time_s": 2.0335181808739438, "epoch": 13, "flow_mse": 0.06287266602739693, "global_step": 880, "grad_norm_max": 1.0403690338134766, "hidden_mse": 0.1264244217425585, "loss": 0.07551510813646019, "lr_blocks": 6.873032967079562e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 68.46298551559448}
|
| 265 |
+
{"avg_micro_time_s": 1.6593307598377578, "epoch": 13, "flow_mse": 0.0616169007960707, "global_step": 890, "grad_norm_max": 1.445999264717102, "hidden_mse": 0.12434439333155752, "loss": 0.07405133983120323, "lr_blocks": 6.7918397477265e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 68.46298551559448}
|
| 266 |
+
{"avg_micro_time_s": 1.6712814146303572, "epoch": 13, "flow_mse": 0.0653102396056056, "global_step": 900, "grad_norm_max": 2.982330322265625, "hidden_mse": 0.12486500488594175, "loss": 0.07779674059711397, "lr_blocks": 6.710100716628345e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 68.46298551559448}
|
| 267 |
+
{"avg_micro_time_s": 1.9577245332402526, "epoch": 13, "flow_mse": 0.06775598619133234, "global_step": 910, "grad_norm_max": 1.3592380285263062, "hidden_mse": 0.1261634855531156, "loss": 0.08037233473733069, "lr_blocks": 6.627840772285784e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 68.46298551559448}
|
| 268 |
+
{"avg_micro_time_s": 1.87712880110048, "epoch": 13, "flow_mse": 0.06632841904647649, "global_step": 920, "grad_norm_max": 1.6363564729690552, "hidden_mse": 0.12368027167394757, "loss": 0.07869644630700351, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 68.46298551559448}
|
| 269 |
+
{"avg_micro_time_s": 2.105622898535512, "epoch": 14, "flow_mse": 0.0650685330387205, "global_step": 930, "grad_norm_max": 0.9703700542449951, "hidden_mse": 0.12554309023544194, "loss": 0.07762284209020437, "lr_blocks": 6.461858523613684e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 68.46298551559448}
|
| 270 |
+
{"avg_micro_time_s": 1.9442461250553607, "epoch": 14, "flow_mse": 0.06523986770771444, "global_step": 940, "grad_norm_max": 1.2431607246398926, "hidden_mse": 0.12500393418595196, "loss": 0.07774026063270867, "lr_blocks": 6.378186779084996e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 68.46298551559448}
|
| 271 |
+
{"avg_micro_time_s": 1.828690172272036, "epoch": 14, "flow_mse": 0.05963015616871417, "global_step": 950, "grad_norm_max": 1.5364301204681396, "hidden_mse": 0.11931315260007978, "loss": 0.07156147151254118, "lr_blocks": 6.294095225512604e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 68.46298551559448}
|
| 272 |
+
{"avg_micro_time_s": 1.8559695239790017, "epoch": 14, "flow_mse": 0.05979857188649476, "global_step": 960, "grad_norm_max": 1.4135713577270508, "hidden_mse": 0.11980592980980874, "loss": 0.07177916495129466, "lr_blocks": 6.209609477998339e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 68.46298551559448}
|
| 273 |
+
{"avg_micro_time_s": 1.7981327558882185, "epoch": 14, "flow_mse": 0.06091945511288941, "global_step": 970, "grad_norm_max": 1.1532989740371704, "hidden_mse": 0.1190660865046084, "loss": 0.0728260634932667, "lr_blocks": 6.124755271719326e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 68.46298551559448}
|
| 274 |
+
{"avg_micro_time_s": 1.8538059903803514, "epoch": 14, "flow_mse": 0.05959429889917374, "global_step": 980, "grad_norm_max": 1.2059638500213623, "hidden_mse": 0.11662983177229763, "loss": 0.07125728223472834, "lr_blocks": 6.039558454088796e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 68.46298551559448}
|
| 275 |
+
{"avg_micro_time_s": 1.7332944313558984, "epoch": 14, "flow_mse": 0.06722243609838188, "global_step": 990, "grad_norm_max": 1.3793468475341797, "hidden_mse": 0.12186120152473451, "loss": 0.07940855622291565, "lr_blocks": 5.954044976882725e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 68.46298551559448}
|
| 276 |
+
{"avg_micro_time_s": 1.7073448181181448, "epoch": 15, "flow_mse": 0.05920077268965543, "global_step": 1000, "grad_norm_max": 1.3150396347045898, "hidden_mse": 0.11587047157809138, "loss": 0.07078781956806779, "lr_blocks": 5.8682408883346535e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 68.46298551559448}
|
| 277 |
+
{"avg_micro_time_s": 1.8634853277268122, "epoch": 15, "flow_mse": 0.06008068108931184, "global_step": 1010, "grad_norm_max": 1.0294694900512695, "hidden_mse": 0.11746087549254298, "loss": 0.07182676899246872, "lr_blocks": 5.782172325201155e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 68.46298551559448}
|
| 278 |
+
{"avg_micro_time_s": 1.8640984378071153, "epoch": 15, "flow_mse": 0.06041769445873797, "global_step": 1020, "grad_norm_max": 1.2436527013778687, "hidden_mse": 0.11662168577313424, "loss": 0.0720798633992672, "lr_blocks": 5.695865504800328e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 68.46298551559448}
|
| 279 |
+
{"avg_micro_time_s": 2.044428478881309, "epoch": 15, "flow_mse": 0.059971056133508685, "global_step": 1030, "grad_norm_max": 0.8764917850494385, "hidden_mse": 0.11762538189068437, "loss": 0.07173359449952842, "lr_blocks": 5.609346717025738e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 68.46298551559448}
|
| 280 |
+
{"avg_micro_time_s": 1.7723727050048184, "epoch": 15, "flow_mse": 0.058173052780330185, "global_step": 1040, "grad_norm_max": 1.1820777654647827, "hidden_mse": 0.11490522287786008, "loss": 0.06966357519850135, "lr_blocks": 5.522642316338268e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 68.46298551559448}
|
| 281 |
+
{"avg_micro_time_s": 1.9333946266793647, "epoch": 15, "flow_mse": 0.06083732731640339, "global_step": 1050, "grad_norm_max": 0.8588114380836487, "hidden_mse": 0.11753584127873183, "loss": 0.07259091124869883, "lr_blocks": 5.435778713738292e-06, "lr_fusion": 5e-05, "peak_memory_gib": 68.46298551559448}
|
| 282 |
+
{"avg_micro_time_s": 1.9301579607854364, "epoch": 16, "flow_mse": 0.05984115526080132, "global_step": 1060, "grad_norm_max": 0.7564420104026794, "hidden_mse": 0.116309258248657, "loss": 0.07147208084352315, "lr_blocks": 5.348782368720627e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 68.46298551559448}
|
| 283 |
+
{"avg_micro_time_s": 1.6499796213785887, "epoch": 16, "flow_mse": 0.05933821597136557, "global_step": 1070, "grad_norm_max": 1.2188807725906372, "hidden_mse": 0.11562003139406443, "loss": 0.07090021898038686, "lr_blocks": 5.2616797812147205e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 68.46298551559448}
|
| 284 |
+
{"avg_micro_time_s": 1.7709604913383374, "epoch": 16, "flow_mse": 0.05792279005981982, "global_step": 1080, "grad_norm_max": 0.9248772263526917, "hidden_mse": 0.11402276260778309, "loss": 0.0693250666372478, "lr_blocks": 5.174497483512506e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 68.46298551559448}
|
| 285 |
+
{"avg_micro_time_s": 1.8961538497242145, "epoch": 16, "flow_mse": 0.05921817822381854, "global_step": 1090, "grad_norm_max": 0.9744265675544739, "hidden_mse": 0.11493033999577165, "loss": 0.07071121269837022, "lr_blocks": 5.087262032186418e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 68.46298551559448}
|
| 286 |
+
{"avg_micro_time_s": 1.8375497608751177, "epoch": 16, "flow_mse": 0.05829775258898735, "global_step": 1100, "grad_norm_max": 0.9396805763244629, "hidden_mse": 0.11368074417114259, "loss": 0.06966582736931741, "lr_blocks": 5e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 68.46298551559448}
|
| 287 |
+
{"avg_micro_time_s": 1.8890359286408058, "epoch": 16, "flow_mse": 0.056764368247240786, "global_step": 1110, "grad_norm_max": 1.4962784051895142, "hidden_mse": 0.11246938547119499, "loss": 0.06801130706444383, "lr_blocks": 4.9127379678135825e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 68.46298551559448}
|
| 288 |
+
{"avg_micro_time_s": 1.9199464306249867, "epoch": 16, "flow_mse": 0.05812495080754161, "global_step": 1120, "grad_norm_max": 1.0150479078292847, "hidden_mse": 0.11339842304587365, "loss": 0.06946479328908027, "lr_blocks": 4.825502516487497e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 68.46298551559448}
|
| 289 |
+
{"avg_micro_time_s": 1.9528670382205746, "epoch": 17, "flow_mse": 0.0582504840567708, "global_step": 1130, "grad_norm_max": 0.5666173100471497, "hidden_mse": 0.11305413423106075, "loss": 0.0695558977779001, "lr_blocks": 4.738320218785281e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 68.46298551559448}
|
| 290 |
+
{"avg_micro_time_s": 1.8691815458922065, "epoch": 17, "flow_mse": 0.056232606945559385, "global_step": 1140, "grad_norm_max": 1.0409326553344727, "hidden_mse": 0.11095626167953015, "loss": 0.06732823327183723, "lr_blocks": 4.651217631279374e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 68.46298551559448}
|
| 291 |
+
{"avg_micro_time_s": 1.9381392417169991, "epoch": 17, "flow_mse": 0.05920435013249517, "global_step": 1150, "grad_norm_max": 0.8891173005104065, "hidden_mse": 0.11471081618219614, "loss": 0.07067543230950833, "lr_blocks": 4.564221286261709e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 68.46298551559448}
|
| 292 |
+
{"avg_micro_time_s": 1.7007469822870918, "epoch": 17, "flow_mse": 0.05892735007219017, "global_step": 1160, "grad_norm_max": 0.9542156457901001, "hidden_mse": 0.11405815165489913, "loss": 0.07033316562883556, "lr_blocks": 4.477357683661734e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 68.46298551559448}
|
| 293 |
+
{"avg_micro_time_s": 1.813334083685186, "epoch": 17, "flow_mse": 0.05461530075408519, "global_step": 1170, "grad_norm_max": 0.8323894143104553, "hidden_mse": 0.10710006449371577, "loss": 0.0653253072872758, "lr_blocks": 4.390653282974264e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 68.46298551559448}
|
| 294 |
+
{"avg_micro_time_s": 1.7458943290723257, "epoch": 17, "flow_mse": 0.058256217977032065, "global_step": 1180, "grad_norm_max": 1.1402466297149658, "hidden_mse": 0.11414804682135582, "loss": 0.06967102284543217, "lr_blocks": 4.304134495199675e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 68.46298551559448}
|
| 295 |
+
{"avg_micro_time_s": 1.915744307746354, "epoch": 18, "flow_mse": 0.055332719627767805, "global_step": 1190, "grad_norm_max": 0.8607915639877319, "hidden_mse": 0.10889494800940157, "loss": 0.06622221460565925, "lr_blocks": 4.217827674798845e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 68.46298551559448}
|
| 296 |
+
{"avg_micro_time_s": 1.9392635020019953, "epoch": 18, "flow_mse": 0.05797289633192122, "global_step": 1200, "grad_norm_max": 1.8679325580596924, "hidden_mse": 0.11137517839670182, "loss": 0.06911041443236172, "lr_blocks": 4.131759111665349e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 68.46298551559448}
|
| 297 |
+
{"avg_micro_time_s": 1.7942509921369494, "epoch": 18, "flow_mse": 0.056557841086760166, "global_step": 1210, "grad_norm_max": 0.794858992099762, "hidden_mse": 0.11070305658504367, "loss": 0.06762814656831324, "lr_blocks": 4.045955023117276e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 68.46298551559448}
|
| 298 |
+
{"avg_micro_time_s": 1.6715436912636505, "epoch": 18, "flow_mse": 0.0571835795417428, "global_step": 1220, "grad_norm_max": 0.9790372848510742, "hidden_mse": 0.11226286897435785, "loss": 0.06840986674651504, "lr_blocks": 3.960441545911205e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 68.46298551559448}
|
| 299 |
+
{"avg_micro_time_s": 2.107250372883573, "epoch": 18, "flow_mse": 0.0564295522402972, "global_step": 1230, "grad_norm_max": 0.7570212483406067, "hidden_mse": 0.10993242673575879, "loss": 0.06742279478348792, "lr_blocks": 3.875244728280676e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 68.46298551559448}
|
| 300 |
+
{"avg_micro_time_s": 2.047389568622748, "epoch": 18, "flow_mse": 0.05746471900492907, "global_step": 1240, "grad_norm_max": 0.8607873320579529, "hidden_mse": 0.11112378425896169, "loss": 0.06857709758915008, "lr_blocks": 3.790390522001662e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 68.46298551559448}
|
| 301 |
+
{"avg_micro_time_s": 1.5449788220197662, "epoch": 18, "flow_mse": 0.05702267340384424, "global_step": 1250, "grad_norm_max": 0.6349190473556519, "hidden_mse": 0.11191629096865655, "loss": 0.06821430274285377, "lr_blocks": 3.705904774487396e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 68.46298551559448}
|
| 302 |
+
{"avg_micro_time_s": 1.8875254256010523, "epoch": 19, "flow_mse": 0.05772090326063335, "global_step": 1260, "grad_norm_max": 0.5843414664268494, "hidden_mse": 0.11196697549894452, "loss": 0.06891760104335845, "lr_blocks": 3.6218132209150047e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 68.46298551559448}
|
| 303 |
+
{"avg_micro_time_s": 1.8776245697605192, "epoch": 19, "flow_mse": 0.05313115566968918, "global_step": 1270, "grad_norm_max": 0.44912803173065186, "hidden_mse": 0.1067539899609983, "loss": 0.06380655500106514, "lr_blocks": 3.538141476386317e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 68.46298551559448}
|
| 304 |
+
{"avg_micro_time_s": 1.9916057762879065, "epoch": 19, "flow_mse": 0.05610430384986103, "global_step": 1280, "grad_norm_max": 0.7865238785743713, "hidden_mse": 0.10909662246704102, "loss": 0.06701396652497352, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 68.46298551559448}
|
| 305 |
+
{"avg_micro_time_s": 1.819823783946049, "epoch": 19, "flow_mse": 0.05514522660523653, "global_step": 1290, "grad_norm_max": 0.9301948547363281, "hidden_mse": 0.10944940159097315, "loss": 0.06609016689471901, "lr_blocks": 3.372159227714218e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 68.46298551559448}
|
| 306 |
+
{"avg_micro_time_s": 2.057221127783123, "epoch": 19, "flow_mse": 0.056626258650794625, "global_step": 1300, "grad_norm_max": 0.5852092504501343, "hidden_mse": 0.1101443249732256, "loss": 0.06764069120399654, "lr_blocks": 3.289899283371657e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 68.46298551559448}
|
| 307 |
+
{"avg_micro_time_s": 1.6946213100774914, "epoch": 19, "flow_mse": 0.05420284080319107, "global_step": 1310, "grad_norm_max": 0.49697157740592957, "hidden_mse": 0.10914511308073999, "loss": 0.06511735212989152, "lr_blocks": 3.2081602522734987e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 68.46298551559448}
|
| 308 |
+
{"avg_micro_time_s": 1.6702450912911446, "epoch": 19, "flow_mse": 0.05620301379822195, "global_step": 1320, "grad_norm_max": 0.8781030178070068, "hidden_mse": 0.11186566464602948, "loss": 0.067389580514282, "lr_blocks": 3.12696703292044e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 68.46298551559448}
|
| 309 |
+
{"avg_micro_time_s": 1.8671580449008616, "epoch": 20, "flow_mse": 0.057774351164698604, "global_step": 1330, "grad_norm_max": 0.6999236941337585, "hidden_mse": 0.1124896040186286, "loss": 0.06902331141754985, "lr_blocks": 3.0463443575536324e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 68.46298551559448}
|
| 310 |
+
{"avg_micro_time_s": 1.7934378478326836, "epoch": 20, "flow_mse": 0.05308823841623962, "global_step": 1340, "grad_norm_max": 0.6592912673950195, "hidden_mse": 0.10645070374011995, "loss": 0.06373330899514258, "lr_blocks": 2.966316784621e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 68.46298551559448}
|
| 311 |
+
{"avg_micro_time_s": 1.983334714593366, "epoch": 20, "flow_mse": 0.05545833441428841, "global_step": 1350, "grad_norm_max": 0.619492769241333, "hidden_mse": 0.10996899055317044, "loss": 0.06645523337647319, "lr_blocks": 2.886908691296504e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 68.46298551559448}
|
| 312 |
+
{"avg_micro_time_s": 1.707897415009211, "epoch": 20, "flow_mse": 0.05316436188295484, "global_step": 1360, "grad_norm_max": 0.7670770883560181, "hidden_mse": 0.10698874769732357, "loss": 0.06386323706246913, "lr_blocks": 2.8081442660546126e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 68.46298551559448}
|
| 313 |
+
{"avg_micro_time_s": 1.8228236074690358, "epoch": 20, "flow_mse": 0.05519023141823709, "global_step": 1370, "grad_norm_max": 0.5514799356460571, "hidden_mse": 0.10873732827603817, "loss": 0.06606396441347898, "lr_blocks": 2.7300475013022666e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 68.46298551559448}
|
| 314 |
+
{"avg_micro_time_s": 1.8020589397929143, "epoch": 20, "flow_mse": 0.05520334360189736, "global_step": 1380, "grad_norm_max": 0.7075578570365906, "hidden_mse": 0.10943609643727542, "loss": 0.0661469531711191, "lr_blocks": 2.6526421860705474e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 68.46298551559448}
|
| 315 |
+
{"avg_micro_time_s": 2.1456252824849797, "epoch": 21, "flow_mse": 0.055343919666484, "global_step": 1390, "grad_norm_max": 0.4367266297340393, "hidden_mse": 0.10879806661978364, "loss": 0.0662237268872559, "lr_blocks": 2.5759518987683154e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 68.46298551559448}
|
| 316 |
+
{"avg_micro_time_s": 1.9225107472884702, "epoch": 21, "flow_mse": 0.05379870082251728, "global_step": 1400, "grad_norm_max": 0.45527708530426025, "hidden_mse": 0.10747067201882601, "loss": 0.0645457681734115, "lr_blocks": 2.5000000000000015e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 68.46298551559448}
|
| 317 |
+
{"avg_micro_time_s": 1.8355080222332618, "epoch": 21, "flow_mse": 0.05405601593665779, "global_step": 1410, "grad_norm_max": 0.578701376914978, "hidden_mse": 0.10813164105638862, "loss": 0.06486918060109019, "lr_blocks": 2.424809625449729e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 68.46298551559448}
|
| 318 |
+
{"avg_micro_time_s": 1.8781015649496113, "epoch": 21, "flow_mse": 0.05536241387017071, "global_step": 1420, "grad_norm_max": 0.38821011781692505, "hidden_mse": 0.11099598519504071, "loss": 0.06646201252005994, "lr_blocks": 2.3504036788339763e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 68.46298551559448}
|
| 319 |
+
{"avg_micro_time_s": 1.7020483079002589, "epoch": 21, "flow_mse": 0.05298192100599408, "global_step": 1430, "grad_norm_max": 0.5125593543052673, "hidden_mse": 0.10700325099751354, "loss": 0.06368224625475705, "lr_blocks": 2.2768048249248648e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 68.46298551559448}
|
| 320 |
+
{"avg_micro_time_s": 1.8793439952991322, "epoch": 21, "flow_mse": 0.05324047850444913, "global_step": 1440, "grad_norm_max": 0.36654290556907654, "hidden_mse": 0.10668059764429927, "loss": 0.06390853836201131, "lr_blocks": 2.204035482646267e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 68.46298551559448}
|
| 321 |
+
{"avg_micro_time_s": 1.697708303979016, "epoch": 21, "flow_mse": 0.055548487138003116, "global_step": 1450, "grad_norm_max": 0.5792702436447144, "hidden_mse": 0.11123776510357858, "loss": 0.06667226417921483, "lr_blocks": 2.132117818244771e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 68.46298551559448}
|
| 322 |
+
{"avg_micro_time_s": 1.8742496912454953, "epoch": 22, "flow_mse": 0.05278900931589306, "global_step": 1460, "grad_norm_max": 0.41618072986602783, "hidden_mse": 0.10622980045154691, "loss": 0.06341198971495032, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 68.46298551559448}
|
| 323 |
+
{"avg_micro_time_s": 2.0268068581106493, "epoch": 22, "flow_mse": 0.05427227606996894, "global_step": 1470, "grad_norm_max": 0.38854727149009705, "hidden_mse": 0.10826945109292865, "loss": 0.06509922100231051, "lr_blocks": 1.990924884239758e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 68.46298551559448}
|
| 324 |
+
{"avg_micro_time_s": 1.6323719802472625, "epoch": 22, "flow_mse": 0.05504520414397121, "global_step": 1480, "grad_norm_max": 0.42108166217803955, "hidden_mse": 0.11108364090323448, "loss": 0.06615356872789561, "lr_blocks": 1.9216926233717087e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 68.46298551559448}
|
| 325 |
+
{"avg_micro_time_s": 1.7635257252361045, "epoch": 22, "flow_mse": 0.05449590594507754, "global_step": 1490, "grad_norm_max": 0.2980191111564636, "hidden_mse": 0.10923452377319337, "loss": 0.06541935848072171, "lr_blocks": 1.8533980447508138e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 68.46298551559448}
|
| 326 |
+
{"avg_micro_time_s": 1.7368722179773615, "epoch": 22, "flow_mse": 0.05059092580340803, "global_step": 1500, "grad_norm_max": 0.3388434052467346, "hidden_mse": 0.10435767890885472, "loss": 0.06102669364772737, "lr_blocks": 1.7860619515673034e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 68.46298551559448}
|
| 327 |
+
{"avg_micro_time_s": 2.083392089595145, "epoch": 22, "flow_mse": 0.05451004351489246, "global_step": 1510, "grad_norm_max": 0.31937962770462036, "hidden_mse": 0.10862588789314032, "loss": 0.0653726323042065, "lr_blocks": 1.7197048550474643e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 68.46298551559448}
|
| 328 |
+
{"avg_micro_time_s": 1.7850786256254652, "epoch": 23, "flow_mse": 0.05411349283531308, "global_step": 1520, "grad_norm_max": 0.43102946877479553, "hidden_mse": 0.1085999496281147, "loss": 0.06497348803095519, "lr_blocks": 1.6543469682057105e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 68.46298551559448}
|
| 329 |
+
{"avg_micro_time_s": 1.6930420300239348, "epoch": 23, "flow_mse": 0.05289990794844926, "global_step": 1530, "grad_norm_max": 0.3967125415802002, "hidden_mse": 0.10759261148050428, "loss": 0.06365916887298227, "lr_blocks": 1.5900081996875083e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 68.46298551559448}
|
| 330 |
+
{"avg_micro_time_s": 1.8721173314916086, "epoch": 23, "flow_mse": 0.05183863551355899, "global_step": 1540, "grad_norm_max": 0.65227872133255, "hidden_mse": 0.10619828049093485, "loss": 0.062458463944494724, "lr_blocks": 1.5267081477050132e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 68.46298551559448}
|
| 331 |
+
{"avg_micro_time_s": 1.9260684134074837, "epoch": 23, "flow_mse": 0.05517242061905563, "global_step": 1550, "grad_norm_max": 0.3173617720603943, "hidden_mse": 0.11037433873862029, "loss": 0.06620985479094088, "lr_blocks": 1.4644660940672628e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 68.46298551559448}
|
| 332 |
+
{"avg_micro_time_s": 2.00732960163441, "epoch": 23, "flow_mse": 0.052165593765676026, "global_step": 1560, "grad_norm_max": 0.48366814851760864, "hidden_mse": 0.10552244568243624, "loss": 0.06271783891133964, "lr_blocks": 1.4033009983067454e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 68.46298551559448}
|
| 333 |
+
{"avg_micro_time_s": 2.0028102155803937, "epoch": 23, "flow_mse": 0.054785998072475195, "global_step": 1570, "grad_norm_max": 0.32774579524993896, "hidden_mse": 0.1094754220917821, "loss": 0.0657335405703634, "lr_blocks": 1.3432314919041478e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 68.46298551559448}
|
| 334 |
+
{"avg_micro_time_s": 1.8898658655904002, "epoch": 23, "flow_mse": 0.05355181619524956, "global_step": 1580, "grad_norm_max": 0.29386377334594727, "hidden_mse": 0.10809278432279826, "loss": 0.06436109468340874, "lr_blocks": 1.2842758726130283e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 68.46298551559448}
|
| 335 |
+
{"avg_micro_time_s": 1.8055248979391765, "epoch": 24, "flow_mse": 0.05223282501101494, "global_step": 1590, "grad_norm_max": 0.30623888969421387, "hidden_mse": 0.10633350862190127, "loss": 0.06286617605946958, "lr_blocks": 1.22645209888614e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 68.46298551559448}
|
| 336 |
+
{"avg_micro_time_s": 1.697861864778679, "epoch": 24, "flow_mse": 0.05407982333563269, "global_step": 1600, "grad_norm_max": 0.41349685192108154, "hidden_mse": 0.1090579808689654, "loss": 0.06498562130145728, "lr_blocks": 1.1697777844051105e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 68.46298551559448}
|
| 337 |
+
{"avg_micro_time_s": 1.858836678104126, "epoch": 24, "flow_mse": 0.0515467718243599, "global_step": 1610, "grad_norm_max": 0.2459108531475067, "hidden_mse": 0.10587860029190779, "loss": 0.06213463232852519, "lr_blocks": 1.1142701927151456e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 68.46298551559448}
|
| 338 |
+
{"avg_micro_time_s": 1.7442898534587585, "epoch": 24, "flow_mse": 0.05539359417743981, "global_step": 1620, "grad_norm_max": 0.4176762104034424, "hidden_mse": 0.11120789768174291, "loss": 0.06651438400149345, "lr_blocks": 1.0599462319663906e-06, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 68.46298551559448}
|
| 339 |
+
{"avg_micro_time_s": 1.9312700220412808, "epoch": 24, "flow_mse": 0.05194714558310807, "global_step": 1630, "grad_norm_max": 0.27055931091308594, "hidden_mse": 0.10536178648471833, "loss": 0.06248332438990474, "lr_blocks": 1.006822449763537e-06, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 68.46298551559448}
|
| 340 |
+
{"avg_micro_time_s": 1.9848020562625608, "epoch": 24, "flow_mse": 0.05159918488934637, "global_step": 1640, "grad_norm_max": 0.2410484105348587, "hidden_mse": 0.10558492876589298, "loss": 0.06215767813846469, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 68.46298551559448}
|
| 341 |
+
{"avg_micro_time_s": 1.9862558376597008, "epoch": 24, "flow_mse": 0.051943618524819615, "global_step": 1650, "grad_norm_max": 0.19621193408966064, "hidden_mse": 0.10532341394573451, "loss": 0.062475959723815325, "lr_blocks": 9.042397785550405e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 68.46298551559448}
|
| 342 |
+
{"avg_micro_time_s": 1.90772508844384, "epoch": 25, "flow_mse": 0.05241604868788272, "global_step": 1660, "grad_norm_max": 0.24477998912334442, "hidden_mse": 0.10615330189466476, "loss": 0.06303137941285968, "lr_blocks": 8.54812137224792e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 68.46298551559448}
|
| 343 |
+
{"avg_micro_time_s": 2.0734705874245267, "epoch": 25, "flow_mse": 0.052382499817758804, "global_step": 1670, "grad_norm_max": 0.18695969879627228, "hidden_mse": 0.10536184627562761, "loss": 0.06291868472471833, "lr_blocks": 8.066471602728804e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 68.46298551559448}
|
| 344 |
+
{"avg_micro_time_s": 1.784898155926203, "epoch": 25, "flow_mse": 0.05266823512502015, "global_step": 1680, "grad_norm_max": 0.30488482117652893, "hidden_mse": 0.10819421242922544, "loss": 0.0634876563679427, "lr_blocks": 7.597595192178702e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 68.46298551559448}
|
| 345 |
+
{"avg_micro_time_s": 1.7039467210503063, "epoch": 25, "flow_mse": 0.053952160244807604, "global_step": 1690, "grad_norm_max": 0.21721898019313812, "hidden_mse": 0.10940732667222619, "loss": 0.06489289305172861, "lr_blocks": 7.141634964894389e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 68.46298551559448}
|
| 346 |
+
{"avg_micro_time_s": 1.7925323831252173, "epoch": 25, "flow_mse": 0.054602593369781974, "global_step": 1700, "grad_norm_max": 0.2606789171695709, "hidden_mse": 0.11060061100870372, "loss": 0.06566265453584492, "lr_blocks": 6.698729810778065e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 68.46298551559448}
|
| 347 |
+
{"avg_micro_time_s": 1.8506990019581282, "epoch": 25, "flow_mse": 0.05084327077493072, "global_step": 1710, "grad_norm_max": 0.18723498284816742, "hidden_mse": 0.10409033391624689, "loss": 0.06125230453908444, "lr_blocks": 6.269014643030214e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 68.46298551559448}
|
| 348 |
+
{"avg_micro_time_s": 1.696594527331763, "epoch": 26, "flow_mse": 0.04985265554860235, "global_step": 1720, "grad_norm_max": 0.16147896647453308, "hidden_mse": 0.10344723165035248, "loss": 0.060197379020974044, "lr_blocks": 5.852620357053651e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 68.46298551559448}
|
| 349 |
+
{"avg_micro_time_s": 1.7064527680224275, "epoch": 26, "flow_mse": 0.052878752723336225, "global_step": 1730, "grad_norm_max": 0.21210241317749023, "hidden_mse": 0.10743851037696005, "loss": 0.0636226034257561, "lr_blocks": 5.449673790581611e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 68.46298551559448}
|
| 350 |
+
{"avg_micro_time_s": 1.9300253918670933, "epoch": 26, "flow_mse": 0.05500989542342723, "global_step": 1740, "grad_norm_max": 0.18679021298885345, "hidden_mse": 0.11223258469253779, "loss": 0.06623315447941423, "lr_blocks": 5.06029768504166e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 68.46298551559448}
|
| 351 |
+
{"avg_micro_time_s": 1.6863267055538018, "epoch": 26, "flow_mse": 0.05216321973130107, "global_step": 1750, "grad_norm_max": 0.1739465296268463, "hidden_mse": 0.10685156406834723, "loss": 0.06284837638959288, "lr_blocks": 4.6846106481675035e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 68.46298551559448}
|
| 352 |
+
{"avg_micro_time_s": 1.9015966636099622, "epoch": 26, "flow_mse": 0.05238161133602262, "global_step": 1760, "grad_norm_max": 0.16062317788600922, "hidden_mse": 0.1064825899899006, "loss": 0.06302987090311944, "lr_blocks": 4.322727117869951e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 68.46298551559448}
|
| 353 |
+
{"avg_micro_time_s": 1.89506446889136, "epoch": 26, "flow_mse": 0.05142894261516631, "global_step": 1770, "grad_norm_max": 0.19871976971626282, "hidden_mse": 0.10540924789384008, "loss": 0.06196986767463386, "lr_blocks": 3.9747573273779816e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 68.46298551559448}
|
| 354 |
+
{"avg_micro_time_s": 2.140559560073598, "epoch": 26, "flow_mse": 0.052294114232063295, "global_step": 1780, "grad_norm_max": 0.13666212558746338, "hidden_mse": 0.10540376659482718, "loss": 0.06283449097536505, "lr_blocks": 3.6408072716606346e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 68.46298551559448}
|
| 355 |
+
{"avg_micro_time_s": 1.9869108784609126, "epoch": 27, "flow_mse": 0.052665405767038466, "global_step": 1790, "grad_norm_max": 0.1539895087480545, "hidden_mse": 0.1076177385635674, "loss": 0.06342717967927457, "lr_blocks": 3.320978675139919e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 68.46298551559448}
|
| 356 |
+
{"avg_micro_time_s": 1.8483484975920874, "epoch": 27, "flow_mse": 0.05281953318044544, "global_step": 1800, "grad_norm_max": 0.1717279851436615, "hidden_mse": 0.10747445961460472, "loss": 0.06356697916053236, "lr_blocks": 3.015368960704584e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 68.46298551559448}
|
| 357 |
+
{"avg_micro_time_s": 1.7725311637754204, "epoch": 27, "flow_mse": 0.053238363936543465, "global_step": 1810, "grad_norm_max": 0.129792258143425, "hidden_mse": 0.10810153055936099, "loss": 0.06404851726256311, "lr_blocks": 2.724071220034158e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 68.46298551559448}
|
| 358 |
+
{"avg_micro_time_s": 1.9046508581523085, "epoch": 27, "flow_mse": 0.05027513452805579, "global_step": 1820, "grad_norm_max": 0.13140571117401123, "hidden_mse": 0.1037200196646154, "loss": 0.06064713676460087, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 68.46298551559448}
|
| 359 |
+
{"avg_micro_time_s": 1.7696023706230335, "epoch": 27, "flow_mse": 0.053843728452920916, "global_step": 1830, "grad_norm_max": 0.10490740090608597, "hidden_mse": 0.10948641048744322, "loss": 0.06479236944578588, "lr_blocks": 2.1847622018482283e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 68.46298551559448}
|
| 360 |
+
{"avg_micro_time_s": 1.8017618429686992, "epoch": 27, "flow_mse": 0.05273311133496464, "global_step": 1840, "grad_norm_max": 0.1325991004705429, "hidden_mse": 0.10784135181456805, "loss": 0.06351724686101079, "lr_blocks": 1.9369152030840553e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 68.46298551559448}
|
| 361 |
+
{"avg_micro_time_s": 1.8663546495808987, "epoch": 28, "flow_mse": 0.05010635161306709, "global_step": 1850, "grad_norm_max": 0.13209085166454315, "hidden_mse": 0.10319266710430385, "loss": 0.06042561884969473, "lr_blocks": 1.7037086855465902e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 68.46298551559448}
|
| 362 |
+
{"avg_micro_time_s": 2.023102572160133, "epoch": 28, "flow_mse": 0.0542366745416075, "global_step": 1860, "grad_norm_max": 0.09557513147592545, "hidden_mse": 0.10798764936625958, "loss": 0.06503543937578797, "lr_blocks": 1.4852136862001766e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 68.46298551559448}
|
| 363 |
+
{"avg_micro_time_s": 1.9713299808965532, "epoch": 28, "flow_mse": 0.050487085804343225, "global_step": 1870, "grad_norm_max": 0.09084595739841461, "hidden_mse": 0.1046027010306716, "loss": 0.06094735627993941, "lr_blocks": 1.2814967607382433e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 68.46298551559448}
|
| 364 |
+
{"avg_micro_time_s": 1.765306223012158, "epoch": 28, "flow_mse": 0.052992650866508485, "global_step": 1880, "grad_norm_max": 0.1068035289645195, "hidden_mse": 0.10854229601100088, "loss": 0.06384688057005405, "lr_blocks": 1.0926199633097156e-07, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 68.46298551559448}
|
| 365 |
+
{"avg_micro_time_s": 1.8488479020117667, "epoch": 28, "flow_mse": 0.05185321508906782, "global_step": 1890, "grad_norm_max": 0.0908748134970665, "hidden_mse": 0.10693074595183134, "loss": 0.06254628994502127, "lr_blocks": 9.186408276168012e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 68.46298551559448}
|
| 366 |
+
{"avg_micro_time_s": 1.835118718787271, "epoch": 28, "flow_mse": 0.050304826558567585, "global_step": 1900, "grad_norm_max": 0.07472742348909378, "hidden_mse": 0.10459219636395574, "loss": 0.06076404661871493, "lr_blocks": 7.59612349389599e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 68.46298551559448}
|
| 367 |
+
{"avg_micro_time_s": 1.8683127601994785, "epoch": 28, "flow_mse": 0.05247240727767349, "global_step": 1910, "grad_norm_max": 0.0885327011346817, "hidden_mse": 0.1072016378864646, "loss": 0.06319257123395801, "lr_blocks": 6.15582970243117e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 68.46298551559448}
|
| 368 |
+
{"avg_micro_time_s": 1.6088743058397086, "epoch": 29, "flow_mse": 0.05225406056270004, "global_step": 1920, "grad_norm_max": 0.06508878618478775, "hidden_mse": 0.10751305781304837, "loss": 0.06300536664202809, "lr_blocks": 4.865965629214819e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 68.46298551559448}
|
| 369 |
+
{"avg_micro_time_s": 1.6788693653565134, "epoch": 29, "flow_mse": 0.05310070822015405, "global_step": 1930, "grad_norm_max": 0.09013048559427261, "hidden_mse": 0.10843602363020183, "loss": 0.0639443107880652, "lr_blocks": 3.726924179339009e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 68.46298551559448}
|
| 370 |
+
{"avg_micro_time_s": 1.8483045230590507, "epoch": 29, "flow_mse": 0.05225380118936301, "global_step": 1940, "grad_norm_max": 0.05455300584435463, "hidden_mse": 0.10687568383291364, "loss": 0.06294136978685856, "lr_blocks": 2.7390523158633552e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 68.46298551559448}
|
| 371 |
+
{"avg_micro_time_s": 2.07668071983353, "epoch": 29, "flow_mse": 0.052197744604200125, "global_step": 1950, "grad_norm_max": 0.06954554468393326, "hidden_mse": 0.10710220159962774, "loss": 0.06290796506218613, "lr_blocks": 1.9026509541272276e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 68.46298551559448}
|
| 372 |
+
{"avg_micro_time_s": 1.890728423825931, "epoch": 29, "flow_mse": 0.052322149462997916, "global_step": 1960, "grad_norm_max": 0.06443354487419128, "hidden_mse": 0.10821447782218457, "loss": 0.06314359758980573, "lr_blocks": 1.2179748700879013e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 68.46298551559448}
|
| 373 |
+
{"avg_micro_time_s": 1.7953088903203025, "epoch": 29, "flow_mse": 0.05093962624669075, "global_step": 1970, "grad_norm_max": 0.07561659812927246, "hidden_mse": 0.10430180076509715, "loss": 0.06136980666778982, "lr_blocks": 6.852326227130835e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 68.46298551559448}
|
| 374 |
+
{"avg_micro_time_s": 1.842160684411647, "epoch": 29, "flow_mse": 0.053311998723074794, "global_step": 1980, "grad_norm_max": 0.04818754270672798, "hidden_mse": 0.10754476888105274, "loss": 0.06406647614203394, "lr_blocks": 3.0458649045211897e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 68.46298551559448}
|
| 375 |
+
{"avg_micro_time_s": 1.7273795908869944, "epoch": 30, "flow_mse": 0.051652479777112605, "global_step": 1990, "grad_norm_max": 0.061520446091890335, "hidden_mse": 0.10536492271348835, "loss": 0.06218897206708789, "lr_blocks": 7.615242180436521e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 68.46298551559448}
|
| 376 |
+
{"avg_micro_time_s": 1.9548408255825054, "epoch": 30, "flow_mse": 0.05280004227533937, "global_step": 2000, "grad_norm_max": 0.054393548518419266, "hidden_mse": 0.10706793488934636, "loss": 0.06350683607161045, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 68.46298551559448}
|
| 377 |
+
swanlab: 📁 View project at
|
| 378 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 379 |
+
swanlab: 🚀 View run pre...8-prompts1000-seed0-bs32x8-from-scratch at
|
| 380 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/4kxa9v7b
|
| 381 |
+
swanlab: Upload complete: 72320 records
|
checkpoints/predictor_v4_1000_seed0/swanlab/.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
*
|
checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/debug/debug.log
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Diagnostic log bound to file: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/debug/debug.log
|
| 2 |
+
2026-07-25 15:13:30.859 | ERROR | swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST https://api.swanlab.cn/api/house/metrics -> 522 (5106ms) trace:unknown | [ERR] code=unknown code message=unknown error
|
| 3 |
+
2026-07-25 15:13:33.721 | ERROR | swanlab.sdk.internal.pkg.safe:block:101 - record chunk upload failed, record_type='scalar': Traceback (most recent call last):
|
| 4 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/pkg/safe/__init__.py", line 98, in block
|
| 5 |
+
yield
|
| 6 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/core_python/transport/dispatch.py", line 73, in _upload_chunk
|
| 7 |
+
self._sender.upload(record_type, chunk)
|
| 8 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/core_python/transport/sender.py", line 138, in upload
|
| 9 |
+
handler(records)
|
| 10 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/core_python/transport/sender.py", line 179, in upload_scalar
|
| 11 |
+
upload_scalar(self._project_id, self._experiment_id, metrics=metrics)
|
| 12 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/core_python/api/upload.py", line 125, in upload_scalar
|
| 13 |
+
client.post("/house/metrics", data, retries=0)
|
| 14 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/core_python/client/__init__.py", line 59, in post
|
| 15 |
+
return _get_client().post(url, data=data, retries=retries, log_error=log_error)
|
| 16 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/pkg/client/__init__.py", line 110, in post
|
| 17 |
+
return self.request("POST", url, json=data, retries=retries, log_error=log_error)
|
| 18 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/pkg/client/__init__.py", line 103, in request
|
| 19 |
+
resp = self._session.request(method, full_url, **kwargs)
|
| 20 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/pkg/client/session.py", line 92, in request
|
| 21 |
+
return super().request(method, url, *args, **kwargs)
|
| 22 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/requests/sessions.py", line 651, in request
|
| 23 |
+
resp = self.send(prep, **send_kwargs)
|
| 24 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/swanlab/sdk/internal/pkg/client/session.py", line 136, in send
|
| 25 |
+
raise ApiError(response, method=method, trace_id=trace_id, code=error_code, message=error_message)
|
| 26 |
+
swanlab.exceptions.ApiError: API Request Failed: [unknown code] unknown error | Trace ID: unknown | POST https://api.swanlab.cn/api/house/metrics
|
| 27 |
+
2026-07-25 15:13:33.726 | WARNING | swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due to network or server issues, retrying automatically.
|
checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/config.yaml
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
activation_dtype:
|
| 2 |
+
desc: ''
|
| 3 |
+
sort: 58
|
| 4 |
+
value: bfloat16
|
| 5 |
+
batch_size:
|
| 6 |
+
desc: ''
|
| 7 |
+
sort: 17
|
| 8 |
+
value: 32
|
| 9 |
+
beta1:
|
| 10 |
+
desc: ''
|
| 11 |
+
sort: 24
|
| 12 |
+
value: 0.9
|
| 13 |
+
beta2:
|
| 14 |
+
desc: ''
|
| 15 |
+
sort: 25
|
| 16 |
+
value: 0.95
|
| 17 |
+
block_warmup_steps:
|
| 18 |
+
desc: ''
|
| 19 |
+
sort: 28
|
| 20 |
+
value: 100
|
| 21 |
+
blocks_lr:
|
| 22 |
+
desc: ''
|
| 23 |
+
sort: 23
|
| 24 |
+
value: 1.0e-05
|
| 25 |
+
data_root:
|
| 26 |
+
desc: ''
|
| 27 |
+
sort: 2
|
| 28 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 29 |
+
dataset_pairs:
|
| 30 |
+
desc: ''
|
| 31 |
+
sort: 52
|
| 32 |
+
value: 18000
|
| 33 |
+
find_unused_parameters:
|
| 34 |
+
desc: ''
|
| 35 |
+
sort: 34
|
| 36 |
+
value: false
|
| 37 |
+
flow_loss_weight:
|
| 38 |
+
desc: ''
|
| 39 |
+
sort: 30
|
| 40 |
+
value: 1.0
|
| 41 |
+
fp32_trainable_params:
|
| 42 |
+
desc: ''
|
| 43 |
+
sort: 32
|
| 44 |
+
value: true
|
| 45 |
+
fusion_lr:
|
| 46 |
+
desc: ''
|
| 47 |
+
sort: 22
|
| 48 |
+
value: 0.0001
|
| 49 |
+
global_batch_size:
|
| 50 |
+
desc: ''
|
| 51 |
+
sort: 55
|
| 52 |
+
value: 256
|
| 53 |
+
grad_clip:
|
| 54 |
+
desc: ''
|
| 55 |
+
sort: 31
|
| 56 |
+
value: 1.0
|
| 57 |
+
gradient_accumulation_steps:
|
| 58 |
+
desc: ''
|
| 59 |
+
sort: 18
|
| 60 |
+
value: 1
|
| 61 |
+
gradient_checkpointing:
|
| 62 |
+
desc: ''
|
| 63 |
+
sort: 33
|
| 64 |
+
value: false
|
| 65 |
+
hidden_loss_weight:
|
| 66 |
+
desc: ''
|
| 67 |
+
sort: 29
|
| 68 |
+
value: 0.1
|
| 69 |
+
keep_snapshots:
|
| 70 |
+
desc: ''
|
| 71 |
+
sort: 37
|
| 72 |
+
value: 20
|
| 73 |
+
latent_height:
|
| 74 |
+
desc: ''
|
| 75 |
+
sort: 14
|
| 76 |
+
value: 60
|
| 77 |
+
latent_width:
|
| 78 |
+
desc: ''
|
| 79 |
+
sort: 15
|
| 80 |
+
value: 104
|
| 81 |
+
local_attn_size:
|
| 82 |
+
desc: ''
|
| 83 |
+
sort: 12
|
| 84 |
+
value: -1
|
| 85 |
+
log_every:
|
| 86 |
+
desc: ''
|
| 87 |
+
sort: 35
|
| 88 |
+
value: 10
|
| 89 |
+
manifest:
|
| 90 |
+
desc: ''
|
| 91 |
+
sort: 3
|
| 92 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl
|
| 93 |
+
manifest_records:
|
| 94 |
+
desc: ''
|
| 95 |
+
sort: 51
|
| 96 |
+
value: 6000
|
| 97 |
+
max_records:
|
| 98 |
+
desc: ''
|
| 99 |
+
sort: 5
|
| 100 |
+
value: {}
|
| 101 |
+
max_steps:
|
| 102 |
+
desc: ''
|
| 103 |
+
sort: 16
|
| 104 |
+
value: 2000
|
| 105 |
+
num_workers:
|
| 106 |
+
desc: ''
|
| 107 |
+
sort: 19
|
| 108 |
+
value: 0
|
| 109 |
+
output_dir:
|
| 110 |
+
desc: ''
|
| 111 |
+
sort: 4
|
| 112 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0
|
| 113 |
+
parameter_groups:
|
| 114 |
+
desc: ''
|
| 115 |
+
sort: 57
|
| 116 |
+
value:
|
| 117 |
+
feature_fusion.anchor_norm.bias: fusion_residual
|
| 118 |
+
feature_fusion.anchor_norm.weight: fusion_residual
|
| 119 |
+
feature_fusion.current_norm.bias: fusion_residual
|
| 120 |
+
feature_fusion.current_norm.weight: fusion_residual
|
| 121 |
+
feature_fusion.mlp.0.bias: fusion_residual
|
| 122 |
+
feature_fusion.mlp.0.weight: fusion_residual
|
| 123 |
+
feature_fusion.mlp.2.bias: fusion_residual
|
| 124 |
+
feature_fusion.mlp.2.weight: fusion_residual
|
| 125 |
+
feature_fusion.previous_norm.bias: fusion_residual
|
| 126 |
+
feature_fusion.previous_norm.weight: fusion_residual
|
| 127 |
+
predictor_blocks.0.cross_attn.norm_q.weight: blocks
|
| 128 |
+
predictor_blocks.0.cross_attn.o.bias: blocks
|
| 129 |
+
predictor_blocks.0.cross_attn.o.weight: blocks
|
| 130 |
+
predictor_blocks.0.cross_attn.q.bias: blocks
|
| 131 |
+
predictor_blocks.0.cross_attn.q.weight: blocks
|
| 132 |
+
predictor_blocks.0.ffn.0.bias: blocks
|
| 133 |
+
predictor_blocks.0.ffn.0.weight: blocks
|
| 134 |
+
predictor_blocks.0.ffn.2.bias: blocks
|
| 135 |
+
predictor_blocks.0.ffn.2.weight: blocks
|
| 136 |
+
predictor_blocks.0.modulation: blocks
|
| 137 |
+
predictor_blocks.0.norm3.bias: blocks
|
| 138 |
+
predictor_blocks.0.norm3.weight: blocks
|
| 139 |
+
predictor_blocks.0.self_attn.k.bias: blocks
|
| 140 |
+
predictor_blocks.0.self_attn.k.weight: blocks
|
| 141 |
+
predictor_blocks.0.self_attn.norm_k.weight: blocks
|
| 142 |
+
predictor_blocks.0.self_attn.norm_q.weight: blocks
|
| 143 |
+
predictor_blocks.0.self_attn.o.bias: blocks
|
| 144 |
+
predictor_blocks.0.self_attn.o.weight: blocks
|
| 145 |
+
predictor_blocks.0.self_attn.q.bias: blocks
|
| 146 |
+
predictor_blocks.0.self_attn.q.weight: blocks
|
| 147 |
+
predictor_blocks.0.self_attn.v.bias: blocks
|
| 148 |
+
predictor_blocks.0.self_attn.v.weight: blocks
|
| 149 |
+
predictor_blocks.1.cross_attn.norm_q.weight: blocks
|
| 150 |
+
predictor_blocks.1.cross_attn.o.bias: blocks
|
| 151 |
+
predictor_blocks.1.cross_attn.o.weight: blocks
|
| 152 |
+
predictor_blocks.1.cross_attn.q.bias: blocks
|
| 153 |
+
predictor_blocks.1.cross_attn.q.weight: blocks
|
| 154 |
+
predictor_blocks.1.ffn.0.bias: blocks
|
| 155 |
+
predictor_blocks.1.ffn.0.weight: blocks
|
| 156 |
+
predictor_blocks.1.ffn.2.bias: blocks
|
| 157 |
+
predictor_blocks.1.ffn.2.weight: blocks
|
| 158 |
+
predictor_blocks.1.modulation: blocks
|
| 159 |
+
predictor_blocks.1.norm3.bias: blocks
|
| 160 |
+
predictor_blocks.1.norm3.weight: blocks
|
| 161 |
+
predictor_blocks.1.self_attn.k.bias: blocks
|
| 162 |
+
predictor_blocks.1.self_attn.k.weight: blocks
|
| 163 |
+
predictor_blocks.1.self_attn.norm_k.weight: blocks
|
| 164 |
+
predictor_blocks.1.self_attn.norm_q.weight: blocks
|
| 165 |
+
predictor_blocks.1.self_attn.o.bias: blocks
|
| 166 |
+
predictor_blocks.1.self_attn.o.weight: blocks
|
| 167 |
+
predictor_blocks.1.self_attn.q.bias: blocks
|
| 168 |
+
predictor_blocks.1.self_attn.q.weight: blocks
|
| 169 |
+
predictor_blocks.1.self_attn.v.bias: blocks
|
| 170 |
+
predictor_blocks.1.self_attn.v.weight: blocks
|
| 171 |
+
residual_out.bias: fusion_residual
|
| 172 |
+
residual_out.weight: fusion_residual
|
| 173 |
+
persistent_workers:
|
| 174 |
+
desc: ''
|
| 175 |
+
sort: 21
|
| 176 |
+
value: false
|
| 177 |
+
pin_memory:
|
| 178 |
+
desc: ''
|
| 179 |
+
sort: 20
|
| 180 |
+
value: false
|
| 181 |
+
prompt_count:
|
| 182 |
+
desc: ''
|
| 183 |
+
sort: 1
|
| 184 |
+
value: 1000
|
| 185 |
+
resume:
|
| 186 |
+
desc: ''
|
| 187 |
+
sort: 38
|
| 188 |
+
value: {}
|
| 189 |
+
save_every:
|
| 190 |
+
desc: ''
|
| 191 |
+
sort: 36
|
| 192 |
+
value: 100
|
| 193 |
+
scheduler_warmup_steps:
|
| 194 |
+
desc: ''
|
| 195 |
+
sort: 27
|
| 196 |
+
value: 100
|
| 197 |
+
schema_version:
|
| 198 |
+
desc: ''
|
| 199 |
+
sort: 0
|
| 200 |
+
value: self_forcing_predictor_v4_bf16_v1
|
| 201 |
+
sink_size:
|
| 202 |
+
desc: ''
|
| 203 |
+
sort: 13
|
| 204 |
+
value: 0
|
| 205 |
+
source_block_ids:
|
| 206 |
+
desc: ''
|
| 207 |
+
sort: 10
|
| 208 |
+
value:
|
| 209 |
+
- 1
|
| 210 |
+
- 28
|
| 211 |
+
steps_per_epoch_per_rank:
|
| 212 |
+
desc: ''
|
| 213 |
+
sort: 53
|
| 214 |
+
value: 66
|
| 215 |
+
strict_teacher_load:
|
| 216 |
+
desc: ''
|
| 217 |
+
sort: 9
|
| 218 |
+
value: true
|
| 219 |
+
supervision_pairs:
|
| 220 |
+
desc: ''
|
| 221 |
+
sort: 50
|
| 222 |
+
value:
|
| 223 |
+
- - 0
|
| 224 |
+
- 1
|
| 225 |
+
- - 1
|
| 226 |
+
- 2
|
| 227 |
+
- - 2
|
| 228 |
+
- 3
|
| 229 |
+
swanlab_description:
|
| 230 |
+
desc: ''
|
| 231 |
+
sort: 47
|
| 232 |
+
value: Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training
|
| 233 |
+
swanlab_experiment:
|
| 234 |
+
desc: ''
|
| 235 |
+
sort: 46
|
| 236 |
+
value: predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs32x8-from-scratch
|
| 237 |
+
swanlab_mode:
|
| 238 |
+
desc: ''
|
| 239 |
+
sort: 49
|
| 240 |
+
value: cloud
|
| 241 |
+
swanlab_project:
|
| 242 |
+
desc: ''
|
| 243 |
+
sort: 44
|
| 244 |
+
value: Self-Forcing-Predictor-v4
|
| 245 |
+
swanlab_tags:
|
| 246 |
+
desc: ''
|
| 247 |
+
sort: 48
|
| 248 |
+
value:
|
| 249 |
+
- predictor-v4
|
| 250 |
+
- self-forcing
|
| 251 |
+
- wan2.1-1.3b
|
| 252 |
+
- offline-prefeature
|
| 253 |
+
- blocks-1-28
|
| 254 |
+
- 8gpu
|
| 255 |
+
- seed0
|
| 256 |
+
swanlab_workspace:
|
| 257 |
+
desc: ''
|
| 258 |
+
sort: 45
|
| 259 |
+
value: {}
|
| 260 |
+
teacher_checkpoint:
|
| 261 |
+
desc: ''
|
| 262 |
+
sort: 7
|
| 263 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt
|
| 264 |
+
teacher_checkpoint_key:
|
| 265 |
+
desc: ''
|
| 266 |
+
sort: 8
|
| 267 |
+
value: generator_ema
|
| 268 |
+
teacher_model_name:
|
| 269 |
+
desc: ''
|
| 270 |
+
sort: 6
|
| 271 |
+
value: Wan2.1-T2V-1.3B
|
| 272 |
+
timestep_shift:
|
| 273 |
+
desc: ''
|
| 274 |
+
sort: 11
|
| 275 |
+
value: 5.0
|
| 276 |
+
trainable_parameter_dtype:
|
| 277 |
+
desc: ''
|
| 278 |
+
sort: 59
|
| 279 |
+
value: float32
|
| 280 |
+
trainable_parameters:
|
| 281 |
+
desc: ''
|
| 282 |
+
sort: 56
|
| 283 |
+
value: 104684032
|
| 284 |
+
use_swanlab:
|
| 285 |
+
desc: ''
|
| 286 |
+
sort: 43
|
| 287 |
+
value: true
|
| 288 |
+
use_wandb:
|
| 289 |
+
desc: ''
|
| 290 |
+
sort: 39
|
| 291 |
+
value: false
|
| 292 |
+
wandb_entity:
|
| 293 |
+
desc: ''
|
| 294 |
+
sort: 41
|
| 295 |
+
value: {}
|
| 296 |
+
wandb_name:
|
| 297 |
+
desc: ''
|
| 298 |
+
sort: 42
|
| 299 |
+
value: predictor-v4-prefeature-blocks1-28-prompts1000-seed0
|
| 300 |
+
wandb_project:
|
| 301 |
+
desc: ''
|
| 302 |
+
sort: 40
|
| 303 |
+
value: Self-Forcing-Predictor-v4
|
| 304 |
+
weight_decay:
|
| 305 |
+
desc: ''
|
| 306 |
+
sort: 26
|
| 307 |
+
value: 0.01
|
| 308 |
+
world_size:
|
| 309 |
+
desc: ''
|
| 310 |
+
sort: 54
|
| 311 |
+
value: 8
|
checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/requirements.txt
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
accelerate==1.14.0
|
| 2 |
+
aiohappyeyeballs==2.7.1
|
| 3 |
+
aiohttp==3.14.1
|
| 4 |
+
aiosignal==1.4.0
|
| 5 |
+
annotated-doc==0.0.4
|
| 6 |
+
annotated-types==0.7.0
|
| 7 |
+
antlr4-python3-runtime==4.9.3
|
| 8 |
+
anyio==4.14.1
|
| 9 |
+
async-timeout==5.0.1
|
| 10 |
+
attrs==26.1.0
|
| 11 |
+
av==13.1.0
|
| 12 |
+
bidict==0.23.1
|
| 13 |
+
blinker==1.9.0
|
| 14 |
+
certifi==2026.6.17
|
| 15 |
+
cffi==2.0.0
|
| 16 |
+
charset-normalizer==3.4.7
|
| 17 |
+
click==8.4.2
|
| 18 |
+
clip==1.0
|
| 19 |
+
coloredlogs==15.0.1
|
| 20 |
+
contourpy==1.3.2
|
| 21 |
+
cryptography==49.0.0
|
| 22 |
+
cuda-bindings==13.3.1
|
| 23 |
+
cuda-pathfinder==1.5.6
|
| 24 |
+
cuda-toolkit==13.0.2
|
| 25 |
+
cycler==0.12.1
|
| 26 |
+
dashscope==1.26.2
|
| 27 |
+
diffusers==0.31.0
|
| 28 |
+
dominate==2.9.1
|
| 29 |
+
easydict==1.13
|
| 30 |
+
einops==0.8.2
|
| 31 |
+
exceptiongroup==1.3.1
|
| 32 |
+
filelock==3.29.0
|
| 33 |
+
flash_attn==2.8.3.post1
|
| 34 |
+
Flask==3.1.3
|
| 35 |
+
Flask-SocketIO==5.6.1
|
| 36 |
+
flatbuffers==25.12.19
|
| 37 |
+
fonttools==4.63.0
|
| 38 |
+
frozenlist==1.8.0
|
| 39 |
+
fsspec==2026.4.0
|
| 40 |
+
ftfy==6.3.1
|
| 41 |
+
gitdb==4.0.12
|
| 42 |
+
GitPython==3.1.50
|
| 43 |
+
h11==0.16.0
|
| 44 |
+
hf-xet==1.5.1
|
| 45 |
+
httpcore==1.0.9
|
| 46 |
+
httpx==0.28.1
|
| 47 |
+
httpx-sse==0.4.3
|
| 48 |
+
huggingface_hub==1.21.0
|
| 49 |
+
humanfriendly==10.0
|
| 50 |
+
idna==3.18
|
| 51 |
+
ImageIO==2.37.3
|
| 52 |
+
imageio-ffmpeg==0.6.0
|
| 53 |
+
importlib_metadata==9.0.0
|
| 54 |
+
itsdangerous==2.2.0
|
| 55 |
+
Jinja2==3.1.6
|
| 56 |
+
kiwisolver==1.5.0
|
| 57 |
+
lazy-loader==0.5
|
| 58 |
+
lmdb==2.2.1
|
| 59 |
+
Mako==1.3.12
|
| 60 |
+
markdown-it-py==4.2.0
|
| 61 |
+
MarkupSafe==3.0.3
|
| 62 |
+
matplotlib==3.10.9
|
| 63 |
+
mdurl==0.1.2
|
| 64 |
+
ml_dtypes==0.5.4
|
| 65 |
+
mpmath==1.3.0
|
| 66 |
+
multidict==6.7.1
|
| 67 |
+
networkx==3.4.2
|
| 68 |
+
ninja==1.13.0
|
| 69 |
+
numpy==1.24.4
|
| 70 |
+
nvidia-cublas==13.1.1.3
|
| 71 |
+
nvidia-cublas-cu12==12.1.3.1
|
| 72 |
+
nvidia-cuda-cupti==13.0.85
|
| 73 |
+
nvidia-cuda-cupti-cu12==12.1.105
|
| 74 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 75 |
+
nvidia-cuda-nvrtc-cu12==12.1.105
|
| 76 |
+
nvidia-cuda-runtime==13.0.96
|
| 77 |
+
nvidia-cuda-runtime-cu12==12.1.105
|
| 78 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 79 |
+
nvidia-cudnn-cu13==9.20.0.48
|
| 80 |
+
nvidia-cufft==12.0.0.61
|
| 81 |
+
nvidia-cufft-cu12==11.0.2.54
|
| 82 |
+
nvidia-cufile==1.15.1.6
|
| 83 |
+
nvidia-curand==10.4.0.35
|
| 84 |
+
nvidia-curand-cu12==10.3.2.106
|
| 85 |
+
nvidia-cusolver==12.0.4.66
|
| 86 |
+
nvidia-cusolver-cu12==11.4.5.107
|
| 87 |
+
nvidia-cusparse==12.6.3.3
|
| 88 |
+
nvidia-cusparse-cu12==12.1.0.106
|
| 89 |
+
nvidia-cusparselt-cu13==0.8.1
|
| 90 |
+
nvidia-ml-py==13.610.43
|
| 91 |
+
nvidia-nccl-cu12==2.21.5
|
| 92 |
+
nvidia-nccl-cu13==2.29.7
|
| 93 |
+
nvidia-nvjitlink==13.0.88
|
| 94 |
+
nvidia-nvjitlink-cu12==12.9.86
|
| 95 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 96 |
+
nvidia-nvtx==13.0.85
|
| 97 |
+
nvidia-nvtx-cu12==12.1.105
|
| 98 |
+
nvidia-tensorrt==99.0.0
|
| 99 |
+
omegaconf==2.3.1
|
| 100 |
+
onnx==1.22.0
|
| 101 |
+
onnx-ir==0.2.1
|
| 102 |
+
onnxconverter-common==1.16.0
|
| 103 |
+
onnxruntime==1.23.2
|
| 104 |
+
onnxscript==0.7.1
|
| 105 |
+
open_clip_torch==3.3.0
|
| 106 |
+
opencv-python==4.11.0.86
|
| 107 |
+
orjson==3.11.9
|
| 108 |
+
packaging==26.2
|
| 109 |
+
pillow==12.2.0
|
| 110 |
+
pip==26.1.2
|
| 111 |
+
platformdirs==4.10.0
|
| 112 |
+
prettytable==3.18.0
|
| 113 |
+
propcache==0.5.2
|
| 114 |
+
protobuf==6.33.6
|
| 115 |
+
psutil==7.2.2
|
| 116 |
+
pwinput==1.0.3
|
| 117 |
+
pycocotools==2.0.11
|
| 118 |
+
pycparser==3.0
|
| 119 |
+
pycuda==2026.1
|
| 120 |
+
pydantic==2.10.6
|
| 121 |
+
pydantic_core==2.27.2
|
| 122 |
+
pydantic-settings==2.14.2
|
| 123 |
+
pyecharts==2.1.0
|
| 124 |
+
Pygments==2.20.0
|
| 125 |
+
pyparsing==3.3.2
|
| 126 |
+
python-dateutil==2.9.0.post0
|
| 127 |
+
python-dotenv==1.2.2
|
| 128 |
+
python-engineio==4.13.3
|
| 129 |
+
python-socketio==5.16.3
|
| 130 |
+
pytools==2026.1.1
|
| 131 |
+
PyYAML==6.0.3
|
| 132 |
+
regex==2026.6.28
|
| 133 |
+
requests==2.34.2
|
| 134 |
+
rich==15.0.0
|
| 135 |
+
safetensors==0.8.0
|
| 136 |
+
scikit-image==0.25.2
|
| 137 |
+
scipy==1.15.3
|
| 138 |
+
self_forcing==0.0.1
|
| 139 |
+
sentencepiece==0.2.1
|
| 140 |
+
sentry-sdk==2.64.0
|
| 141 |
+
setuptools==81.0.0
|
| 142 |
+
shellingham==1.5.4
|
| 143 |
+
simple-websocket==1.1.0
|
| 144 |
+
simplejson==4.1.1
|
| 145 |
+
siphash24==1.8
|
| 146 |
+
six==1.17.0
|
| 147 |
+
smmap==5.0.3
|
| 148 |
+
starlette==1.3.1
|
| 149 |
+
swanlab==0.9.0
|
| 150 |
+
sympy==1.13.1
|
| 151 |
+
tensorrt==11.1.0.106
|
| 152 |
+
tensorrt_cu13==11.1.0.106
|
| 153 |
+
tensorrt_cu13_bindings==11.1.0.106
|
| 154 |
+
tensorrt_cu13_libs==11.1.0.106
|
| 155 |
+
tifffile==2025.5.10
|
| 156 |
+
timm==1.0.27
|
| 157 |
+
tokenizers==0.22.2
|
| 158 |
+
torch==2.5.1+cu121
|
| 159 |
+
torchao==0.17.0
|
| 160 |
+
torchvision==0.20.1+cu121
|
| 161 |
+
tqdm==4.68.3
|
| 162 |
+
transformers==5.12.1
|
| 163 |
+
triton==3.1.0
|
| 164 |
+
typer==0.25.1
|
| 165 |
+
typing_extensions==4.15.0
|
| 166 |
+
typing-inspection==0.4.2
|
| 167 |
+
urllib3==2.7.0
|
| 168 |
+
wandb==0.28.0
|
| 169 |
+
watchdog==6.0.0
|
| 170 |
+
wcwidth==0.8.2
|
| 171 |
+
websocket-client==1.9.0
|
| 172 |
+
Werkzeug==3.1.8
|
| 173 |
+
wheel==0.47.0
|
| 174 |
+
wrapt==2.2.2
|
| 175 |
+
wsproto==1.3.2
|
| 176 |
+
yarl==1.24.2
|
| 177 |
+
zipp==4.1.0
|
checkpoints/predictor_v4_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b/files/swanlab-metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_version":2,"hardware":{"apple_silicon":null,"cpu":{"brand":"AMD EPYC 7R13 Processor","physical_count":96,"logical_count":192},"memory":{"total":1999,"total_unit":"GB"},"accelerators":[{"vendor":"nvidia","devices":[{"index":0,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":1,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":2,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":3,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":4,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":5,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":6,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":7,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"}],"version":"580.167.08","cuda_version":"13.0","cann_version":null}]},"runtime":{"os":"Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.34","os_pretty":"Amazon Linux 2023.12.20260611","hostname":"ip-172-31-49-199.us-west-2.compute.internal","pid":3380635,"cwd":"/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing","python_version":"3.10.20","python_verbose":"3.10.20 | packaged by conda-forge | (main, Jun 11 2026, 03:31:56) [GCC 14.3.0]","python_executable":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python","command":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python -u train.py --config_path /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/predictor_v4.yaml --disable-wandb"},"git":{"remote_url":"https://github.com/guandeh17/Self-Forcing","branch":"main","commit":"33593df3e81fa3ec10239271dd2c100facac6de1"},"swanlab":{"version":"0.9.0","run_dir":"/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/run-20260725_144159-4kxa9v7b"}}
|
checkpoints/predictor_v4_1000_seed0/sync_watcher.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
等待 end-to-end supervisor PID=1251561 完成。
|
| 2 |
+
Supervisor 已退出，检查完整产物后开始同步。
|
| 3 |
+
错误：找不到 NVMe 源目录：/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0
|
checkpoints/predictor_v4_1000_seed0/train_config.json
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_dtype": "bfloat16",
|
| 3 |
+
"batch_size": 32,
|
| 4 |
+
"beta1": 0.9,
|
| 5 |
+
"beta2": 0.95,
|
| 6 |
+
"block_warmup_steps": 100,
|
| 7 |
+
"blocks_lr": 1e-05,
|
| 8 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 9 |
+
"dataset_pairs": 18000,
|
| 10 |
+
"find_unused_parameters": false,
|
| 11 |
+
"flow_loss_weight": 1.0,
|
| 12 |
+
"fp32_trainable_params": true,
|
| 13 |
+
"fusion_lr": 0.0001,
|
| 14 |
+
"global_batch_size": 256,
|
| 15 |
+
"grad_clip": 1.0,
|
| 16 |
+
"gradient_accumulation_steps": 1,
|
| 17 |
+
"gradient_checkpointing": false,
|
| 18 |
+
"hidden_loss_weight": 0.1,
|
| 19 |
+
"keep_snapshots": 20,
|
| 20 |
+
"latent_height": 60,
|
| 21 |
+
"latent_width": 104,
|
| 22 |
+
"local_attn_size": -1,
|
| 23 |
+
"log_every": 10,
|
| 24 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 25 |
+
"manifest_records": 6000,
|
| 26 |
+
"max_records": null,
|
| 27 |
+
"max_steps": 2000,
|
| 28 |
+
"num_workers": 0,
|
| 29 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 30 |
+
"parameter_groups": {
|
| 31 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 32 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 33 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 34 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 35 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 36 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 37 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 38 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 39 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 40 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 41 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 42 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 43 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 44 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 45 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 46 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 47 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 48 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 49 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 50 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 51 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 52 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 53 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 54 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 55 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 56 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 57 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 58 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 59 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 60 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 61 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 62 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 63 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 64 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 65 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 66 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 67 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 68 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 69 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 70 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 71 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 72 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 73 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 74 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 75 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 76 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 77 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 78 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 79 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 80 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 81 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 82 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 83 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 84 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 85 |
+
"residual_out.bias": "fusion_residual",
|
| 86 |
+
"residual_out.weight": "fusion_residual"
|
| 87 |
+
},
|
| 88 |
+
"persistent_workers": false,
|
| 89 |
+
"pin_memory": false,
|
| 90 |
+
"prompt_count": 1000,
|
| 91 |
+
"resume": null,
|
| 92 |
+
"save_every": 100,
|
| 93 |
+
"scheduler_warmup_steps": 100,
|
| 94 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 95 |
+
"sink_size": 0,
|
| 96 |
+
"source_block_ids": [
|
| 97 |
+
1,
|
| 98 |
+
28
|
| 99 |
+
],
|
| 100 |
+
"steps_per_epoch_per_rank": 66,
|
| 101 |
+
"strict_teacher_load": true,
|
| 102 |
+
"supervision_pairs": [
|
| 103 |
+
[
|
| 104 |
+
0,
|
| 105 |
+
1
|
| 106 |
+
],
|
| 107 |
+
[
|
| 108 |
+
1,
|
| 109 |
+
2
|
| 110 |
+
],
|
| 111 |
+
[
|
| 112 |
+
2,
|
| 113 |
+
3
|
| 114 |
+
]
|
| 115 |
+
],
|
| 116 |
+
"swanlab_description": "Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training",
|
| 117 |
+
"swanlab_experiment": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs32x8-from-scratch",
|
| 118 |
+
"swanlab_mode": "cloud",
|
| 119 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 120 |
+
"swanlab_run_id": "4kxa9v7b",
|
| 121 |
+
"swanlab_tags": [
|
| 122 |
+
"predictor-v4",
|
| 123 |
+
"self-forcing",
|
| 124 |
+
"wan2.1-1.3b",
|
| 125 |
+
"offline-prefeature",
|
| 126 |
+
"blocks-1-28",
|
| 127 |
+
"8gpu",
|
| 128 |
+
"seed0"
|
| 129 |
+
],
|
| 130 |
+
"swanlab_workspace": null,
|
| 131 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 132 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 133 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 134 |
+
"timestep_shift": 5.0,
|
| 135 |
+
"trainable_parameter_dtype": "float32",
|
| 136 |
+
"trainable_parameters": 104684032,
|
| 137 |
+
"use_swanlab": true,
|
| 138 |
+
"use_wandb": false,
|
| 139 |
+
"wandb_entity": null,
|
| 140 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 141 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 142 |
+
"weight_decay": 0.01,
|
| 143 |
+
"world_size": 8
|
| 144 |
+
}
|
checkpoints/predictor_v4_1000_seed0/train_log.jsonl
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"avg_micro_time_s": 7.782912929877057, "epoch": 0, "flow_mse": 1.728674739599228, "global_step": 1, "grad_norm_max": 1.224261999130249, "hidden_mse": 0.34973957017064095, "loss": 1.7636487036943436, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 59.944087982177734}
|
| 2 |
+
{"avg_micro_time_s": 1.852714978063078, "epoch": 0, "flow_mse": 1.7325291699833338, "global_step": 10, "grad_norm_max": 1.5914783477783203, "hidden_mse": 0.36041637013355887, "loss": 1.7685708122120962, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 68.46298551559448}
|
| 3 |
+
{"avg_micro_time_s": 1.846399891503097, "epoch": 0, "flow_mse": 1.699481675028801, "global_step": 20, "grad_norm_max": 1.9605422019958496, "hidden_mse": 0.38648081049323085, "loss": 1.7381297543644907, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 4 |
+
{"avg_micro_time_s": 2.076498315717618, "epoch": 0, "flow_mse": 1.5910302087664605, "global_step": 30, "grad_norm_max": 1.6393431425094604, "hidden_mse": 0.6486873880028725, "loss": 1.6558989509940147, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 5 |
+
{"avg_micro_time_s": 1.8261107150639875, "epoch": 0, "flow_mse": 1.4074327528476716, "global_step": 40, "grad_norm_max": 1.6300629377365112, "hidden_mse": 1.5101009473204614, "loss": 1.5584428489208222, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 6 |
+
{"avg_micro_time_s": 1.7557343540451258, "epoch": 0, "flow_mse": 1.3375679016113282, "global_step": 50, "grad_norm_max": 1.179707407951355, "hidden_mse": 1.7423540219664575, "loss": 1.5118033051490785, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 68.46298551559448}
|
| 7 |
+
{"avg_micro_time_s": 1.9954431485108217, "epoch": 0, "flow_mse": 1.288495972752571, "global_step": 60, "grad_norm_max": 0.9528158903121948, "hidden_mse": 1.6803042054176331, "loss": 1.4565263986587524, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 8 |
+
{"avg_micro_time_s": 2.073848069201631, "epoch": 1, "flow_mse": 1.2329863071441651, "global_step": 70, "grad_norm_max": 2.1396095752716064, "hidden_mse": 1.468385949730873, "loss": 1.3798249036073686, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 9 |
+
{"avg_micro_time_s": 1.9025867714255584, "epoch": 1, "flow_mse": 1.1653944417834283, "global_step": 80, "grad_norm_max": 3.283475875854492, "hidden_mse": 1.3008124545216562, "loss": 1.2954756870865822, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 10 |
+
{"avg_micro_time_s": 1.8846276455253248, "epoch": 1, "flow_mse": 1.0928176268935204, "global_step": 90, "grad_norm_max": 2.976677179336548, "hidden_mse": 1.1887190774083138, "loss": 1.211689528822899, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 11 |
+
{"avg_micro_time_s": 2.0524098566529574, "epoch": 1, "flow_mse": 1.0091206416487695, "global_step": 100, "grad_norm_max": 2.0720155239105225, "hidden_mse": 1.1053067579865457, "loss": 1.119651308655739, "lr_blocks": 1.0000000000000001e-07, "lr_fusion": 0.0001, "peak_memory_gib": 68.46298551559448}
|
| 12 |
+
{"avg_micro_time_s": 1.8157492470301806, "epoch": 1, "flow_mse": 0.9281245060265064, "global_step": 110, "grad_norm_max": 3.5477075576782227, "hidden_mse": 1.0355910658836365, "loss": 1.031683611869812, "lr_blocks": 1.1e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 68.46298551559448}
|
| 13 |
+
{"avg_micro_time_s": 1.7559494119937882, "epoch": 1, "flow_mse": 0.8380454987287522, "global_step": 120, "grad_norm_max": 3.569852828979492, "hidden_mse": 1.0150623381137849, "loss": 0.9395517356693746, "lr_blocks": 2.1000000000000002e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 68.46298551559448}
|
| 14 |
+
{"avg_micro_time_s": 1.9936403229308781, "epoch": 1, "flow_mse": 0.7462111793458462, "global_step": 130, "grad_norm_max": 2.9960243701934814, "hidden_mse": 0.9869170427322388, "loss": 0.8449028812348843, "lr_blocks": 3.1000000000000004e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 68.46298551559448}
|
| 15 |
+
{"avg_micro_time_s": 1.8761029311470339, "epoch": 2, "flow_mse": 0.6552131280303002, "global_step": 140, "grad_norm_max": 3.033362627029419, "hidden_mse": 0.9761701941490174, "loss": 0.7528301522135735, "lr_blocks": 4.1e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 68.46298551559448}
|
| 16 |
+
{"avg_micro_time_s": 1.6636652560366203, "epoch": 2, "flow_mse": 0.5641707576811313, "global_step": 150, "grad_norm_max": 3.3895790576934814, "hidden_mse": 0.9558152981102467, "loss": 0.6597522847354412, "lr_blocks": 5.1e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 68.46298551559448}
|
| 17 |
+
{"avg_micro_time_s": 1.8629546527619825, "epoch": 2, "flow_mse": 0.49576797895133495, "global_step": 160, "grad_norm_max": 4.9101152420043945, "hidden_mse": 0.9376181185245515, "loss": 0.5895297914743424, "lr_blocks": 6.1e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 68.46298551559448}
|
| 18 |
+
{"avg_micro_time_s": 1.865570171797299, "epoch": 2, "flow_mse": 0.44515644647181035, "global_step": 170, "grad_norm_max": 3.2292189598083496, "hidden_mse": 0.9383152291178704, "loss": 0.5389879673719407, "lr_blocks": 7.100000000000001e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 68.46298551559448}
|
| 19 |
+
{"avg_micro_time_s": 2.0450479036619074, "epoch": 2, "flow_mse": 0.38247467912733557, "global_step": 180, "grad_norm_max": 2.7824766635894775, "hidden_mse": 0.9554134920239449, "loss": 0.47801603078842164, "lr_blocks": 8.1e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 68.46298551559448}
|
| 20 |
+
{"avg_micro_time_s": 2.016261681757169, "epoch": 2, "flow_mse": 0.3393749196082354, "global_step": 190, "grad_norm_max": 2.3986711502075195, "hidden_mse": 0.9583184562623501, "loss": 0.4352067667990923, "lr_blocks": 9.100000000000001e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 68.46298551559448}
|
| 21 |
+
{"avg_micro_time_s": 1.955125979850709, "epoch": 3, "flow_mse": 0.2995503291487694, "global_step": 200, "grad_norm_max": 2.834411144256592, "hidden_mse": 0.9311102323234082, "loss": 0.39266135394573215, "lr_blocks": 1e-05, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 68.46298551559448}
|
| 22 |
+
{"avg_micro_time_s": 1.9450525438660407, "epoch": 3, "flow_mse": 0.26215458251535895, "global_step": 210, "grad_norm_max": 2.5127832889556885, "hidden_mse": 0.9175012007355691, "loss": 0.35390470623970033, "lr_blocks": 9.999238475781957e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 68.46298551559448}
|
| 23 |
+
{"avg_micro_time_s": 1.794988799588464, "epoch": 3, "flow_mse": 0.24451670069247486, "global_step": 220, "grad_norm_max": 3.86234712600708, "hidden_mse": 0.8481401547789574, "loss": 0.32933071814477444, "lr_blocks": 9.99695413509548e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 68.46298551559448}
|
| 24 |
+
{"avg_micro_time_s": 1.821383213158697, "epoch": 3, "flow_mse": 0.21927626319229604, "global_step": 230, "grad_norm_max": 2.894408702850342, "hidden_mse": 0.7950924828648568, "loss": 0.2987855151295662, "lr_blocks": 9.993147673772869e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 68.46298551559448}
|
| 25 |
+
{"avg_micro_time_s": 1.7578791950494634, "epoch": 3, "flow_mse": 0.21038782224059105, "global_step": 240, "grad_norm_max": 4.587767124176025, "hidden_mse": 0.7239097714424134, "loss": 0.2827788006514311, "lr_blocks": 9.987820251299121e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 68.46298551559448}
|
| 26 |
+
{"avg_micro_time_s": 1.9880367784600823, "epoch": 3, "flow_mse": 0.18143612183630467, "global_step": 250, "grad_norm_max": 2.522402048110962, "hidden_mse": 0.6568384729325771, "loss": 0.24711996987462045, "lr_blocks": 9.980973490458728e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 68.46298551559448}
|
| 27 |
+
{"avg_micro_time_s": 2.1574960700992962, "epoch": 3, "flow_mse": 0.17069688346236944, "global_step": 260, "grad_norm_max": 2.6453685760498047, "hidden_mse": 0.5946614742279053, "loss": 0.23016303274780514, "lr_blocks": 9.972609476841368e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 68.46298551559448}
|
| 28 |
+
{"avg_micro_time_s": 1.7216765362391016, "epoch": 4, "flow_mse": 0.16511701364070178, "global_step": 270, "grad_norm_max": 4.101485252380371, "hidden_mse": 0.5324683826416731, "loss": 0.21836385373026135, "lr_blocks": 9.962730758206612e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 68.46298551559448}
|
| 29 |
+
{"avg_micro_time_s": 2.036255760942004, "epoch": 4, "flow_mse": 0.15243881475180387, "global_step": 280, "grad_norm_max": 3.777620792388916, "hidden_mse": 0.49607212990522387, "loss": 0.20204602926969528, "lr_blocks": 9.951340343707852e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 68.46298551559448}
|
| 30 |
+
{"avg_micro_time_s": 2.026448777728365, "epoch": 4, "flow_mse": 0.13280713185667992, "global_step": 290, "grad_norm_max": 2.864901542663574, "hidden_mse": 0.45720995105803014, "loss": 0.1785281287506223, "lr_blocks": 9.938441702975689e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 68.46298551559448}
|
| 31 |
+
{"avg_micro_time_s": 1.7816235546153623, "epoch": 4, "flow_mse": 0.1316212204284966, "global_step": 300, "grad_norm_max": 3.557399272918701, "hidden_mse": 0.4236030507832766, "loss": 0.17398152723908425, "lr_blocks": 9.924038765061042e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 68.46298551559448}
|
| 32 |
+
{"avg_micro_time_s": 1.7509075272682821, "epoch": 4, "flow_mse": 0.12680668467655778, "global_step": 310, "grad_norm_max": 2.9080326557159424, "hidden_mse": 0.3970677562057972, "loss": 0.16651346255093813, "lr_blocks": 9.908135917238321e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 68.46298551559448}
|
| 33 |
+
{"avg_micro_time_s": 1.94211900010705, "epoch": 4, "flow_mse": 0.1265168657526374, "global_step": 320, "grad_norm_max": 3.0853631496429443, "hidden_mse": 0.3776782535016537, "loss": 0.16428469233214857, "lr_blocks": 9.890738003669029e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 68.46298551559448}
|
| 34 |
+
{"avg_micro_time_s": 1.9201412342023105, "epoch": 4, "flow_mse": 0.11589663326740265, "global_step": 330, "grad_norm_max": 3.0462610721588135, "hidden_mse": 0.3677100505679846, "loss": 0.1526676405221224, "lr_blocks": 9.871850323926178e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 68.46298551559448}
|
| 35 |
+
{"avg_micro_time_s": 1.8802110167060164, "epoch": 5, "flow_mse": 0.10986252082511783, "global_step": 340, "grad_norm_max": 2.5844221115112305, "hidden_mse": 0.3474765542894602, "loss": 0.14461017716675997, "lr_blocks": 9.851478631379982e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 68.46298551559448}
|
| 36 |
+
{"avg_micro_time_s": 1.970779871637933, "epoch": 5, "flow_mse": 0.10572945503517986, "global_step": 350, "grad_norm_max": 2.6753783226013184, "hidden_mse": 0.33289663605391984, "loss": 0.13901912048459053, "lr_blocks": 9.829629131445342e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 68.46298551559448}
|
| 37 |
+
{"avg_micro_time_s": 1.8511785630384112, "epoch": 5, "flow_mse": 0.09914825893938542, "global_step": 360, "grad_norm_max": 2.757209539413452, "hidden_mse": 0.3192139830440283, "loss": 0.13106965916231275, "lr_blocks": 9.806308479691595e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 68.46298551559448}
|
| 38 |
+
{"avg_micro_time_s": 1.9199401149497135, "epoch": 5, "flow_mse": 0.0996528490446508, "global_step": 370, "grad_norm_max": 2.9345085620880127, "hidden_mse": 0.3086499966681004, "loss": 0.1305178500711918, "lr_blocks": 9.781523779815178e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 68.46298551559448}
|
| 39 |
+
{"avg_micro_time_s": 1.7542048408882693, "epoch": 5, "flow_mse": 0.11152176447212697, "global_step": 380, "grad_norm_max": 3.106746196746826, "hidden_mse": 0.30627303943037987, "loss": 0.1421490689739585, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 68.46298551559448}
|
| 40 |
+
{"avg_micro_time_s": 1.9700474178520382, "epoch": 5, "flow_mse": 0.10044981110841036, "global_step": 390, "grad_norm_max": 2.560649871826172, "hidden_mse": 0.2952960841357708, "loss": 0.1299794196151197, "lr_blocks": 9.727592877996585e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 68.46298551559448}
|
| 41 |
+
{"avg_micro_time_s": 1.6663271824698314, "epoch": 6, "flow_mse": 0.09705260461196304, "global_step": 400, "grad_norm_max": 2.532449722290039, "hidden_mse": 0.29162609074264767, "loss": 0.1262152137234807, "lr_blocks": 9.698463103929542e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 68.46298551559448}
|
| 42 |
+
{"avg_micro_time_s": 1.904068420661497, "epoch": 6, "flow_mse": 0.09287562295794488, "global_step": 410, "grad_norm_max": 2.16715145111084, "hidden_mse": 0.28314792811870576, "loss": 0.121190416906029, "lr_blocks": 9.667902132486009e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 68.46298551559448}
|
| 43 |
+
{"avg_micro_time_s": 1.744727985771897, "epoch": 6, "flow_mse": 0.09119896441698075, "global_step": 420, "grad_norm_max": 2.299543619155884, "hidden_mse": 0.275063331797719, "loss": 0.1187052981927991, "lr_blocks": 9.635919272833938e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 68.46298551559448}
|
| 44 |
+
{"avg_micro_time_s": 1.7956668802158675, "epoch": 6, "flow_mse": 0.08759928597137333, "global_step": 430, "grad_norm_max": 2.317425489425659, "hidden_mse": 0.2642714731395245, "loss": 0.11402643444016576, "lr_blocks": 9.602524267262202e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 68.46298551559448}
|
| 45 |
+
{"avg_micro_time_s": 1.9157415576424683, "epoch": 6, "flow_mse": 0.08734634332358837, "global_step": 440, "grad_norm_max": 2.760629415512085, "hidden_mse": 0.25739369336515666, "loss": 0.11308571305125953, "lr_blocks": 9.567727288213005e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 68.46298551559448}
|
| 46 |
+
{"avg_micro_time_s": 1.8787613131338732, "epoch": 6, "flow_mse": 0.08993547437712551, "global_step": 450, "grad_norm_max": 3.014104127883911, "hidden_mse": 0.25076083578169345, "loss": 0.11501155886799097, "lr_blocks": 9.531538935183252e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 68.46298551559448}
|
| 47 |
+
{"avg_micro_time_s": 1.8499236255913276, "epoch": 6, "flow_mse": 0.0833338218741119, "global_step": 460, "grad_norm_max": 2.0357143878936768, "hidden_mse": 0.24471132159233094, "loss": 0.10780495461076499, "lr_blocks": 9.493970231495836e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 68.46298551559448}
|
| 48 |
+
{"avg_micro_time_s": 1.8277902282425202, "epoch": 7, "flow_mse": 0.08892108229920269, "global_step": 470, "grad_norm_max": 2.870374917984009, "hidden_mse": 0.24110151771456004, "loss": 0.11303123468533159, "lr_blocks": 9.45503262094184e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 68.46298551559448}
|
| 49 |
+
{"avg_micro_time_s": 1.8054274757610984, "epoch": 7, "flow_mse": 0.08921196823939681, "global_step": 480, "grad_norm_max": 2.4050748348236084, "hidden_mse": 0.24109693542122843, "loss": 0.113321662414819, "lr_blocks": 9.414737964294636e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 50 |
+
{"avg_micro_time_s": 1.969536912314652, "epoch": 7, "flow_mse": 0.08481648499146105, "global_step": 490, "grad_norm_max": 4.0341668128967285, "hidden_mse": 0.23553607985377312, "loss": 0.10837009344249965, "lr_blocks": 9.37309853569698e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 68.46298551559448}
|
| 51 |
+
{"avg_micro_time_s": 1.7191957566945348, "epoch": 7, "flow_mse": 0.08171517765149475, "global_step": 500, "grad_norm_max": 2.7221522331237793, "hidden_mse": 0.22536225505173207, "loss": 0.10425140429288149, "lr_blocks": 9.330127018922195e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 68.46298551559448}
|
| 52 |
+
{"avg_micro_time_s": 1.8753900891068043, "epoch": 7, "flow_mse": 0.08210981907323003, "global_step": 510, "grad_norm_max": 1.9840606451034546, "hidden_mse": 0.22030116170644762, "loss": 0.10413993652909995, "lr_blocks": 9.285836503510562e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 68.46298551559448}
|
| 53 |
+
{"avg_micro_time_s": 1.9803101174184121, "epoch": 7, "flow_mse": 0.08212232678197325, "global_step": 520, "grad_norm_max": 2.06720232963562, "hidden_mse": 0.21767251621931794, "loss": 0.10388957895338535, "lr_blocks": 9.24024048078213e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 68.46298551559448}
|
| 54 |
+
{"avg_micro_time_s": 1.7816847848371253, "epoch": 8, "flow_mse": 0.07924596662633121, "global_step": 530, "grad_norm_max": 2.3816592693328857, "hidden_mse": 0.21423672661185267, "loss": 0.10066964011639357, "lr_blocks": 9.193352839727122e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 68.46298551559448}
|
| 55 |
+
{"avg_micro_time_s": 1.862245862412965, "epoch": 8, "flow_mse": 0.07841471419669688, "global_step": 540, "grad_norm_max": 2.3286426067352295, "hidden_mse": 0.2106100643053651, "loss": 0.09947572173550726, "lr_blocks": 9.145187862775208e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 68.46298551559448}
|
| 56 |
+
{"avg_micro_time_s": 2.0419705960011925, "epoch": 8, "flow_mse": 0.07392775514163077, "global_step": 550, "grad_norm_max": 1.3650901317596436, "hidden_mse": 0.2003183823078871, "loss": 0.09395959414541721, "lr_blocks": 9.09576022144496e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 68.46298551559448}
|
| 57 |
+
{"avg_micro_time_s": 1.6736296482136823, "epoch": 8, "flow_mse": 0.07821477656252683, "global_step": 560, "grad_norm_max": 2.5323493480682373, "hidden_mse": 0.19945396259427073, "loss": 0.09816017346456647, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 68.46298551559448}
|
| 58 |
+
{"avg_micro_time_s": 2.003066141484305, "epoch": 8, "flow_mse": 0.08001610483042897, "global_step": 570, "grad_norm_max": 2.0001752376556396, "hidden_mse": 0.19836203493177892, "loss": 0.09985230900347233, "lr_blocks": 8.993177550236464e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 68.46298551559448}
|
| 59 |
+
{"avg_micro_time_s": 1.9908761504004362, "epoch": 8, "flow_mse": 0.07824829332530499, "global_step": 580, "grad_norm_max": 2.1289918422698975, "hidden_mse": 0.19441615901887419, "loss": 0.09768990930169821, "lr_blocks": 8.94005376803361e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 68.46298551559448}
|
| 60 |
+
{"avg_micro_time_s": 1.722032713617955, "epoch": 8, "flow_mse": 0.07608461757190527, "global_step": 590, "grad_norm_max": 2.0547220706939697, "hidden_mse": 0.19045738708227875, "loss": 0.09513035686686636, "lr_blocks": 8.885729807284855e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 68.46298551559448}
|
| 61 |
+
{"avg_micro_time_s": 1.5744449160105434, "epoch": 9, "flow_mse": 0.07611198453232647, "global_step": 600, "grad_norm_max": 1.646276831626892, "hidden_mse": 0.1878601059317589, "loss": 0.09489799588918686, "lr_blocks": 8.83022221559489e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 68.46298551559448}
|
| 62 |
+
{"avg_micro_time_s": 1.9617489955402563, "epoch": 9, "flow_mse": 0.07234360408037901, "global_step": 610, "grad_norm_max": 2.1270275115966797, "hidden_mse": 0.18017836939543486, "loss": 0.09036144157871605, "lr_blocks": 8.773547901113862e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 68.46298551559448}
|
| 63 |
+
{"avg_micro_time_s": 1.899630580819212, "epoch": 9, "flow_mse": 0.07547258702106774, "global_step": 620, "grad_norm_max": 2.535357713699341, "hidden_mse": 0.17847571074962618, "loss": 0.09332015877589583, "lr_blocks": 8.715724127386971e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 68.46298551559448}
|
| 64 |
+
{"avg_micro_time_s": 1.9009491362783593, "epoch": 9, "flow_mse": 0.07615209170617164, "global_step": 630, "grad_norm_max": 2.885998487472534, "hidden_mse": 0.17413493599742652, "loss": 0.09356558537110687, "lr_blocks": 8.656768508095853e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 68.46298551559448}
|
| 65 |
+
{"avg_micro_time_s": 2.033017601488973, "epoch": 9, "flow_mse": 0.07332893046550452, "global_step": 640, "grad_norm_max": 1.3615809679031372, "hidden_mse": 0.17425470147281885, "loss": 0.09075440168380738, "lr_blocks": 8.596699001693257e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 68.46298551559448}
|
| 66 |
+
{"avg_micro_time_s": 1.6646606730137137, "epoch": 9, "flow_mse": 0.07165623712353408, "global_step": 650, "grad_norm_max": 2.018874406814575, "hidden_mse": 0.17005239613354206, "loss": 0.08866147724911572, "lr_blocks": 8.535533905932739e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 68.46298551559448}
|
| 67 |
+
{"avg_micro_time_s": 2.01382767639152, "epoch": 9, "flow_mse": 0.07549470965750515, "global_step": 660, "grad_norm_max": 1.9090533256530762, "hidden_mse": 0.1704067310318351, "loss": 0.0925353833474219, "lr_blocks": 8.473291852294986e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 68.46298551559448}
|
| 68 |
+
{"avg_micro_time_s": 2.044479904185573, "epoch": 10, "flow_mse": 0.07384380707517267, "global_step": 670, "grad_norm_max": 1.6318471431732178, "hidden_mse": 0.16633677743375302, "loss": 0.09047748530283571, "lr_blocks": 8.409991800312493e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 68.46298551559448}
|
| 69 |
+
{"avg_micro_time_s": 1.994344039564021, "epoch": 10, "flow_mse": 0.07081504138186574, "global_step": 680, "grad_norm_max": 2.8086788654327393, "hidden_mse": 0.16047870106995107, "loss": 0.08686291193589568, "lr_blocks": 8.345653031794292e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 68.46298551559448}
|
| 70 |
+
{"avg_micro_time_s": 1.7797691442654469, "epoch": 10, "flow_mse": 0.07098656282760203, "global_step": 690, "grad_norm_max": 1.6218206882476807, "hidden_mse": 0.15934279719367625, "loss": 0.086920843180269, "lr_blocks": 8.280295144952537e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 68.46298551559448}
|
| 71 |
+
{"avg_micro_time_s": 1.8434631714044372, "epoch": 10, "flow_mse": 0.07692104973830283, "global_step": 700, "grad_norm_max": 1.9783955812454224, "hidden_mse": 0.16042201202362777, "loss": 0.0929632511921227, "lr_blocks": 8.213938048432697e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 68.46298551559448}
|
| 72 |
+
{"avg_micro_time_s": 1.8933369300531924, "epoch": 10, "flow_mse": 0.06859853272326291, "global_step": 710, "grad_norm_max": 1.2764592170715332, "hidden_mse": 0.15258000474423172, "loss": 0.08385653356090189, "lr_blocks": 8.146601955249187e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 68.46298551559448}
|
| 73 |
+
{"avg_micro_time_s": 1.863453762442805, "epoch": 10, "flow_mse": 0.0677291950210929, "global_step": 720, "grad_norm_max": 2.756103992462158, "hidden_mse": 0.15030786208808422, "loss": 0.08275998155586423, "lr_blocks": 8.078307376628292e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 68.46298551559448}
|
| 74 |
+
{"avg_micro_time_s": 1.5886019194789698, "epoch": 11, "flow_mse": 0.06930862842127682, "global_step": 730, "grad_norm_max": 2.032028913497925, "hidden_mse": 0.14793342733755707, "loss": 0.08410197151824833, "lr_blocks": 8.009075115760243e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 68.46298551559448}
|
| 75 |
+
{"avg_micro_time_s": 1.7562155689141947, "epoch": 11, "flow_mse": 0.0670456328894943, "global_step": 740, "grad_norm_max": 1.5815403461456299, "hidden_mse": 0.14644069485366346, "loss": 0.08168970262631775, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 68.46298551559448}
|
| 76 |
+
{"avg_micro_time_s": 1.9935554676223548, "epoch": 11, "flow_mse": 0.06759789921343327, "global_step": 750, "grad_norm_max": 1.5702470541000366, "hidden_mse": 0.1456328984349966, "loss": 0.08216118961572648, "lr_blocks": 7.86788218175523e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 68.46298551559448}
|
| 77 |
+
{"avg_micro_time_s": 1.7392390850000083, "epoch": 11, "flow_mse": 0.0679298588540405, "global_step": 760, "grad_norm_max": 1.7798969745635986, "hidden_mse": 0.145010856539011, "loss": 0.0824309445451945, "lr_blocks": 7.795964517353734e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 68.46298551559448}
|
| 78 |
+
{"avg_micro_time_s": 1.883487955624878, "epoch": 11, "flow_mse": 0.06899964096955956, "global_step": 770, "grad_norm_max": 2.0383424758911133, "hidden_mse": 0.14355416363105178, "loss": 0.08335505775175989, "lr_blocks": 7.723195175075136e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 68.46298551559448}
|
| 79 |
+
{"avg_micro_time_s": 1.902795129803417, "epoch": 11, "flow_mse": 0.06839722348377109, "global_step": 780, "grad_norm_max": 1.341465950012207, "hidden_mse": 0.14199498109519482, "loss": 0.08259672177955509, "lr_blocks": 7.649596321166024e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 68.46298551559448}
|
| 80 |
+
{"avg_micro_time_s": 1.899291706715303, "epoch": 11, "flow_mse": 0.06829775697551668, "global_step": 790, "grad_norm_max": 1.6974772214889526, "hidden_mse": 0.14167611030861735, "loss": 0.08246536822989584, "lr_blocks": 7.575190374550272e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 68.46298551559448}
|
| 81 |
+
{"avg_micro_time_s": 1.9210877921635985, "epoch": 12, "flow_mse": 0.06724138637073339, "global_step": 800, "grad_norm_max": 1.6519050598144531, "hidden_mse": 0.13736099740490318, "loss": 0.08097748612053693, "lr_blocks": 7.500000000000001e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 68.46298551559448}
|
| 82 |
+
{"avg_micro_time_s": 1.839930806370103, "epoch": 12, "flow_mse": 0.06486202948726714, "global_step": 810, "grad_norm_max": 1.9736137390136719, "hidden_mse": 0.13331869533285498, "loss": 0.07819389933720232, "lr_blocks": 7.424048101231687e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 68.46298551559448}
|
| 83 |
+
{"avg_micro_time_s": 1.7203275968320668, "epoch": 12, "flow_mse": 0.06997489500790835, "global_step": 820, "grad_norm_max": 1.8273866176605225, "hidden_mse": 0.1383979432284832, "loss": 0.0838146897032857, "lr_blocks": 7.347357813929455e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 68.46298551559448}
|
| 84 |
+
{"avg_micro_time_s": 1.8308625894540456, "epoch": 12, "flow_mse": 0.06103807487525046, "global_step": 830, "grad_norm_max": 1.24065363407135, "hidden_mse": 0.12885685674846173, "loss": 0.07392376055940986, "lr_blocks": 7.269952498697734e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 68.46298551559448}
|
| 85 |
+
{"avg_micro_time_s": 1.951346460581408, "epoch": 12, "flow_mse": 0.0672075655311346, "global_step": 840, "grad_norm_max": 1.4299647808074951, "hidden_mse": 0.1341157573275268, "loss": 0.08061914164572954, "lr_blocks": 7.191855733945388e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 68.46298551559448}
|
| 86 |
+
{"avg_micro_time_s": 1.9852554764482193, "epoch": 12, "flow_mse": 0.06508104703389109, "global_step": 850, "grad_norm_max": 1.3047852516174316, "hidden_mse": 0.13088483419269323, "loss": 0.07816953058354557, "lr_blocks": 7.113091308703498e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 68.46298551559448}
|
| 87 |
+
{"avg_micro_time_s": 1.8463190050082632, "epoch": 13, "flow_mse": 0.06611472489312291, "global_step": 860, "grad_norm_max": 1.3949657678604126, "hidden_mse": 0.13323927726596593, "loss": 0.07943865223787726, "lr_blocks": 7.033683215379002e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 88 |
+
{"avg_micro_time_s": 1.8538904657267268, "epoch": 13, "flow_mse": 0.0621907118242234, "global_step": 870, "grad_norm_max": 1.0381734371185303, "hidden_mse": 0.1276529028080404, "loss": 0.07495600231923163, "lr_blocks": 6.953655642446368e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 68.46298551559448}
|
| 89 |
+
{"avg_micro_time_s": 2.0335181808739438, "epoch": 13, "flow_mse": 0.06287266602739693, "global_step": 880, "grad_norm_max": 1.0403690338134766, "hidden_mse": 0.1264244217425585, "loss": 0.07551510813646019, "lr_blocks": 6.873032967079562e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 68.46298551559448}
|
| 90 |
+
{"avg_micro_time_s": 1.6593307598377578, "epoch": 13, "flow_mse": 0.0616169007960707, "global_step": 890, "grad_norm_max": 1.445999264717102, "hidden_mse": 0.12434439333155752, "loss": 0.07405133983120323, "lr_blocks": 6.7918397477265e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 68.46298551559448}
|
| 91 |
+
{"avg_micro_time_s": 1.6712814146303572, "epoch": 13, "flow_mse": 0.0653102396056056, "global_step": 900, "grad_norm_max": 2.982330322265625, "hidden_mse": 0.12486500488594175, "loss": 0.07779674059711397, "lr_blocks": 6.710100716628345e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 68.46298551559448}
|
| 92 |
+
{"avg_micro_time_s": 1.9577245332402526, "epoch": 13, "flow_mse": 0.06775598619133234, "global_step": 910, "grad_norm_max": 1.3592380285263062, "hidden_mse": 0.1261634855531156, "loss": 0.08037233473733069, "lr_blocks": 6.627840772285784e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 68.46298551559448}
|
| 93 |
+
{"avg_micro_time_s": 1.87712880110048, "epoch": 13, "flow_mse": 0.06632841904647649, "global_step": 920, "grad_norm_max": 1.6363564729690552, "hidden_mse": 0.12368027167394757, "loss": 0.07869644630700351, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 68.46298551559448}
|
| 94 |
+
{"avg_micro_time_s": 2.105622898535512, "epoch": 14, "flow_mse": 0.0650685330387205, "global_step": 930, "grad_norm_max": 0.9703700542449951, "hidden_mse": 0.12554309023544194, "loss": 0.07762284209020437, "lr_blocks": 6.461858523613684e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 68.46298551559448}
|
| 95 |
+
{"avg_micro_time_s": 1.9442461250553607, "epoch": 14, "flow_mse": 0.06523986770771444, "global_step": 940, "grad_norm_max": 1.2431607246398926, "hidden_mse": 0.12500393418595196, "loss": 0.07774026063270867, "lr_blocks": 6.378186779084996e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 68.46298551559448}
|
| 96 |
+
{"avg_micro_time_s": 1.828690172272036, "epoch": 14, "flow_mse": 0.05963015616871417, "global_step": 950, "grad_norm_max": 1.5364301204681396, "hidden_mse": 0.11931315260007978, "loss": 0.07156147151254118, "lr_blocks": 6.294095225512604e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 68.46298551559448}
|
| 97 |
+
{"avg_micro_time_s": 1.8559695239790017, "epoch": 14, "flow_mse": 0.05979857188649476, "global_step": 960, "grad_norm_max": 1.4135713577270508, "hidden_mse": 0.11980592980980874, "loss": 0.07177916495129466, "lr_blocks": 6.209609477998339e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 68.46298551559448}
|
| 98 |
+
{"avg_micro_time_s": 1.7981327558882185, "epoch": 14, "flow_mse": 0.06091945511288941, "global_step": 970, "grad_norm_max": 1.1532989740371704, "hidden_mse": 0.1190660865046084, "loss": 0.0728260634932667, "lr_blocks": 6.124755271719326e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 68.46298551559448}
|
| 99 |
+
{"avg_micro_time_s": 1.8538059903803514, "epoch": 14, "flow_mse": 0.05959429889917374, "global_step": 980, "grad_norm_max": 1.2059638500213623, "hidden_mse": 0.11662983177229763, "loss": 0.07125728223472834, "lr_blocks": 6.039558454088796e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 68.46298551559448}
|
| 100 |
+
{"avg_micro_time_s": 1.7332944313558984, "epoch": 14, "flow_mse": 0.06722243609838188, "global_step": 990, "grad_norm_max": 1.3793468475341797, "hidden_mse": 0.12186120152473451, "loss": 0.07940855622291565, "lr_blocks": 5.954044976882725e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 68.46298551559448}
|
| 101 |
+
{"avg_micro_time_s": 1.7073448181181448, "epoch": 15, "flow_mse": 0.05920077268965543, "global_step": 1000, "grad_norm_max": 1.3150396347045898, "hidden_mse": 0.11587047157809138, "loss": 0.07078781956806779, "lr_blocks": 5.8682408883346535e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 68.46298551559448}
|
| 102 |
+
{"avg_micro_time_s": 1.8634853277268122, "epoch": 15, "flow_mse": 0.06008068108931184, "global_step": 1010, "grad_norm_max": 1.0294694900512695, "hidden_mse": 0.11746087549254298, "loss": 0.07182676899246872, "lr_blocks": 5.782172325201155e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 68.46298551559448}
|
| 103 |
+
{"avg_micro_time_s": 1.8640984378071153, "epoch": 15, "flow_mse": 0.06041769445873797, "global_step": 1020, "grad_norm_max": 1.2436527013778687, "hidden_mse": 0.11662168577313424, "loss": 0.0720798633992672, "lr_blocks": 5.695865504800328e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 68.46298551559448}
|
| 104 |
+
{"avg_micro_time_s": 2.044428478881309, "epoch": 15, "flow_mse": 0.059971056133508685, "global_step": 1030, "grad_norm_max": 0.8764917850494385, "hidden_mse": 0.11762538189068437, "loss": 0.07173359449952842, "lr_blocks": 5.609346717025738e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 68.46298551559448}
|
| 105 |
+
{"avg_micro_time_s": 1.7723727050048184, "epoch": 15, "flow_mse": 0.058173052780330185, "global_step": 1040, "grad_norm_max": 1.1820777654647827, "hidden_mse": 0.11490522287786008, "loss": 0.06966357519850135, "lr_blocks": 5.522642316338268e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 68.46298551559448}
|
| 106 |
+
{"avg_micro_time_s": 1.9333946266793647, "epoch": 15, "flow_mse": 0.06083732731640339, "global_step": 1050, "grad_norm_max": 0.8588114380836487, "hidden_mse": 0.11753584127873183, "loss": 0.07259091124869883, "lr_blocks": 5.435778713738292e-06, "lr_fusion": 5e-05, "peak_memory_gib": 68.46298551559448}
|
| 107 |
+
{"avg_micro_time_s": 1.9301579607854364, "epoch": 16, "flow_mse": 0.05984115526080132, "global_step": 1060, "grad_norm_max": 0.7564420104026794, "hidden_mse": 0.116309258248657, "loss": 0.07147208084352315, "lr_blocks": 5.348782368720627e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 68.46298551559448}
|
| 108 |
+
{"avg_micro_time_s": 1.6499796213785887, "epoch": 16, "flow_mse": 0.05933821597136557, "global_step": 1070, "grad_norm_max": 1.2188807725906372, "hidden_mse": 0.11562003139406443, "loss": 0.07090021898038686, "lr_blocks": 5.2616797812147205e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 68.46298551559448}
|
| 109 |
+
{"avg_micro_time_s": 1.7709604913383374, "epoch": 16, "flow_mse": 0.05792279005981982, "global_step": 1080, "grad_norm_max": 0.9248772263526917, "hidden_mse": 0.11402276260778309, "loss": 0.0693250666372478, "lr_blocks": 5.174497483512506e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 68.46298551559448}
|
| 110 |
+
{"avg_micro_time_s": 1.8961538497242145, "epoch": 16, "flow_mse": 0.05921817822381854, "global_step": 1090, "grad_norm_max": 0.9744265675544739, "hidden_mse": 0.11493033999577165, "loss": 0.07071121269837022, "lr_blocks": 5.087262032186418e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 68.46298551559448}
|
| 111 |
+
{"avg_micro_time_s": 1.8375497608751177, "epoch": 16, "flow_mse": 0.05829775258898735, "global_step": 1100, "grad_norm_max": 0.9396805763244629, "hidden_mse": 0.11368074417114259, "loss": 0.06966582736931741, "lr_blocks": 5e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 68.46298551559448}
|
| 112 |
+
{"avg_micro_time_s": 1.8890359286408058, "epoch": 16, "flow_mse": 0.056764368247240786, "global_step": 1110, "grad_norm_max": 1.4962784051895142, "hidden_mse": 0.11246938547119499, "loss": 0.06801130706444383, "lr_blocks": 4.9127379678135825e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 68.46298551559448}
|
| 113 |
+
{"avg_micro_time_s": 1.9199464306249867, "epoch": 16, "flow_mse": 0.05812495080754161, "global_step": 1120, "grad_norm_max": 1.0150479078292847, "hidden_mse": 0.11339842304587365, "loss": 0.06946479328908027, "lr_blocks": 4.825502516487497e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 68.46298551559448}
|
| 114 |
+
{"avg_micro_time_s": 1.9528670382205746, "epoch": 17, "flow_mse": 0.0582504840567708, "global_step": 1130, "grad_norm_max": 0.5666173100471497, "hidden_mse": 0.11305413423106075, "loss": 0.0695558977779001, "lr_blocks": 4.738320218785281e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 68.46298551559448}
|
| 115 |
+
{"avg_micro_time_s": 1.8691815458922065, "epoch": 17, "flow_mse": 0.056232606945559385, "global_step": 1140, "grad_norm_max": 1.0409326553344727, "hidden_mse": 0.11095626167953015, "loss": 0.06732823327183723, "lr_blocks": 4.651217631279374e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 68.46298551559448}
|
| 116 |
+
{"avg_micro_time_s": 1.9381392417169991, "epoch": 17, "flow_mse": 0.05920435013249517, "global_step": 1150, "grad_norm_max": 0.8891173005104065, "hidden_mse": 0.11471081618219614, "loss": 0.07067543230950833, "lr_blocks": 4.564221286261709e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 68.46298551559448}
|
| 117 |
+
{"avg_micro_time_s": 1.7007469822870918, "epoch": 17, "flow_mse": 0.05892735007219017, "global_step": 1160, "grad_norm_max": 0.9542156457901001, "hidden_mse": 0.11405815165489913, "loss": 0.07033316562883556, "lr_blocks": 4.477357683661734e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 68.46298551559448}
|
| 118 |
+
{"avg_micro_time_s": 1.813334083685186, "epoch": 17, "flow_mse": 0.05461530075408519, "global_step": 1170, "grad_norm_max": 0.8323894143104553, "hidden_mse": 0.10710006449371577, "loss": 0.0653253072872758, "lr_blocks": 4.390653282974264e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 68.46298551559448}
|
| 119 |
+
{"avg_micro_time_s": 1.7458943290723257, "epoch": 17, "flow_mse": 0.058256217977032065, "global_step": 1180, "grad_norm_max": 1.1402466297149658, "hidden_mse": 0.11414804682135582, "loss": 0.06967102284543217, "lr_blocks": 4.304134495199675e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 68.46298551559448}
|
| 120 |
+
{"avg_micro_time_s": 1.915744307746354, "epoch": 18, "flow_mse": 0.055332719627767805, "global_step": 1190, "grad_norm_max": 0.8607915639877319, "hidden_mse": 0.10889494800940157, "loss": 0.06622221460565925, "lr_blocks": 4.217827674798845e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 68.46298551559448}
|
| 121 |
+
{"avg_micro_time_s": 1.9392635020019953, "epoch": 18, "flow_mse": 0.05797289633192122, "global_step": 1200, "grad_norm_max": 1.8679325580596924, "hidden_mse": 0.11137517839670182, "loss": 0.06911041443236172, "lr_blocks": 4.131759111665349e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 68.46298551559448}
|
| 122 |
+
{"avg_micro_time_s": 1.7942509921369494, "epoch": 18, "flow_mse": 0.056557841086760166, "global_step": 1210, "grad_norm_max": 0.794858992099762, "hidden_mse": 0.11070305658504367, "loss": 0.06762814656831324, "lr_blocks": 4.045955023117276e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 68.46298551559448}
|
| 123 |
+
{"avg_micro_time_s": 1.6715436912636505, "epoch": 18, "flow_mse": 0.0571835795417428, "global_step": 1220, "grad_norm_max": 0.9790372848510742, "hidden_mse": 0.11226286897435785, "loss": 0.06840986674651504, "lr_blocks": 3.960441545911205e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 68.46298551559448}
|
| 124 |
+
{"avg_micro_time_s": 2.107250372883573, "epoch": 18, "flow_mse": 0.0564295522402972, "global_step": 1230, "grad_norm_max": 0.7570212483406067, "hidden_mse": 0.10993242673575879, "loss": 0.06742279478348792, "lr_blocks": 3.875244728280676e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 68.46298551559448}
|
| 125 |
+
{"avg_micro_time_s": 2.047389568622748, "epoch": 18, "flow_mse": 0.05746471900492907, "global_step": 1240, "grad_norm_max": 0.8607873320579529, "hidden_mse": 0.11112378425896169, "loss": 0.06857709758915008, "lr_blocks": 3.790390522001662e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 68.46298551559448}
|
| 126 |
+
{"avg_micro_time_s": 1.5449788220197662, "epoch": 18, "flow_mse": 0.05702267340384424, "global_step": 1250, "grad_norm_max": 0.6349190473556519, "hidden_mse": 0.11191629096865655, "loss": 0.06821430274285377, "lr_blocks": 3.705904774487396e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 68.46298551559448}
|
| 127 |
+
{"avg_micro_time_s": 1.8875254256010523, "epoch": 19, "flow_mse": 0.05772090326063335, "global_step": 1260, "grad_norm_max": 0.5843414664268494, "hidden_mse": 0.11196697549894452, "loss": 0.06891760104335845, "lr_blocks": 3.6218132209150047e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 68.46298551559448}
|
| 128 |
+
{"avg_micro_time_s": 1.8776245697605192, "epoch": 19, "flow_mse": 0.05313115566968918, "global_step": 1270, "grad_norm_max": 0.44912803173065186, "hidden_mse": 0.1067539899609983, "loss": 0.06380655500106514, "lr_blocks": 3.538141476386317e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 68.46298551559448}
|
| 129 |
+
{"avg_micro_time_s": 1.9916057762879065, "epoch": 19, "flow_mse": 0.05610430384986103, "global_step": 1280, "grad_norm_max": 0.7865238785743713, "hidden_mse": 0.10909662246704102, "loss": 0.06701396652497352, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 68.46298551559448}
|
| 130 |
+
{"avg_micro_time_s": 1.819823783946049, "epoch": 19, "flow_mse": 0.05514522660523653, "global_step": 1290, "grad_norm_max": 0.9301948547363281, "hidden_mse": 0.10944940159097315, "loss": 0.06609016689471901, "lr_blocks": 3.372159227714218e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 68.46298551559448}
|
| 131 |
+
{"avg_micro_time_s": 2.057221127783123, "epoch": 19, "flow_mse": 0.056626258650794625, "global_step": 1300, "grad_norm_max": 0.5852092504501343, "hidden_mse": 0.1101443249732256, "loss": 0.06764069120399654, "lr_blocks": 3.289899283371657e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 68.46298551559448}
|
| 132 |
+
{"avg_micro_time_s": 1.6946213100774914, "epoch": 19, "flow_mse": 0.05420284080319107, "global_step": 1310, "grad_norm_max": 0.49697157740592957, "hidden_mse": 0.10914511308073999, "loss": 0.06511735212989152, "lr_blocks": 3.2081602522734987e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 68.46298551559448}
|
| 133 |
+
{"avg_micro_time_s": 1.6702450912911446, "epoch": 19, "flow_mse": 0.05620301379822195, "global_step": 1320, "grad_norm_max": 0.8781030178070068, "hidden_mse": 0.11186566464602948, "loss": 0.067389580514282, "lr_blocks": 3.12696703292044e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 68.46298551559448}
|
| 134 |
+
{"avg_micro_time_s": 1.8671580449008616, "epoch": 20, "flow_mse": 0.057774351164698604, "global_step": 1330, "grad_norm_max": 0.6999236941337585, "hidden_mse": 0.1124896040186286, "loss": 0.06902331141754985, "lr_blocks": 3.0463443575536324e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 68.46298551559448}
|
| 135 |
+
{"avg_micro_time_s": 1.7934378478326836, "epoch": 20, "flow_mse": 0.05308823841623962, "global_step": 1340, "grad_norm_max": 0.6592912673950195, "hidden_mse": 0.10645070374011995, "loss": 0.06373330899514258, "lr_blocks": 2.966316784621e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 68.46298551559448}
|
| 136 |
+
{"avg_micro_time_s": 1.983334714593366, "epoch": 20, "flow_mse": 0.05545833441428841, "global_step": 1350, "grad_norm_max": 0.619492769241333, "hidden_mse": 0.10996899055317044, "loss": 0.06645523337647319, "lr_blocks": 2.886908691296504e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 68.46298551559448}
|
| 137 |
+
{"avg_micro_time_s": 1.707897415009211, "epoch": 20, "flow_mse": 0.05316436188295484, "global_step": 1360, "grad_norm_max": 0.7670770883560181, "hidden_mse": 0.10698874769732357, "loss": 0.06386323706246913, "lr_blocks": 2.8081442660546126e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 68.46298551559448}
|
| 138 |
+
{"avg_micro_time_s": 1.8228236074690358, "epoch": 20, "flow_mse": 0.05519023141823709, "global_step": 1370, "grad_norm_max": 0.5514799356460571, "hidden_mse": 0.10873732827603817, "loss": 0.06606396441347898, "lr_blocks": 2.7300475013022666e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 68.46298551559448}
|
| 139 |
+
{"avg_micro_time_s": 1.8020589397929143, "epoch": 20, "flow_mse": 0.05520334360189736, "global_step": 1380, "grad_norm_max": 0.7075578570365906, "hidden_mse": 0.10943609643727542, "loss": 0.0661469531711191, "lr_blocks": 2.6526421860705474e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 68.46298551559448}
|
| 140 |
+
{"avg_micro_time_s": 2.1456252824849797, "epoch": 21, "flow_mse": 0.055343919666484, "global_step": 1390, "grad_norm_max": 0.4367266297340393, "hidden_mse": 0.10879806661978364, "loss": 0.0662237268872559, "lr_blocks": 2.5759518987683154e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 68.46298551559448}
|
| 141 |
+
{"avg_micro_time_s": 1.9225107472884702, "epoch": 21, "flow_mse": 0.05379870082251728, "global_step": 1400, "grad_norm_max": 0.45527708530426025, "hidden_mse": 0.10747067201882601, "loss": 0.0645457681734115, "lr_blocks": 2.5000000000000015e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 68.46298551559448}
|
| 142 |
+
{"avg_micro_time_s": 1.8355080222332618, "epoch": 21, "flow_mse": 0.05405601593665779, "global_step": 1410, "grad_norm_max": 0.578701376914978, "hidden_mse": 0.10813164105638862, "loss": 0.06486918060109019, "lr_blocks": 2.424809625449729e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 68.46298551559448}
|
| 143 |
+
{"avg_micro_time_s": 1.8781015649496113, "epoch": 21, "flow_mse": 0.05536241387017071, "global_step": 1420, "grad_norm_max": 0.38821011781692505, "hidden_mse": 0.11099598519504071, "loss": 0.06646201252005994, "lr_blocks": 2.3504036788339763e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 68.46298551559448}
|
| 144 |
+
{"avg_micro_time_s": 1.7020483079002589, "epoch": 21, "flow_mse": 0.05298192100599408, "global_step": 1430, "grad_norm_max": 0.5125593543052673, "hidden_mse": 0.10700325099751354, "loss": 0.06368224625475705, "lr_blocks": 2.2768048249248648e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 68.46298551559448}
|
| 145 |
+
{"avg_micro_time_s": 1.8793439952991322, "epoch": 21, "flow_mse": 0.05324047850444913, "global_step": 1440, "grad_norm_max": 0.36654290556907654, "hidden_mse": 0.10668059764429927, "loss": 0.06390853836201131, "lr_blocks": 2.204035482646267e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 68.46298551559448}
|
| 146 |
+
{"avg_micro_time_s": 1.697708303979016, "epoch": 21, "flow_mse": 0.055548487138003116, "global_step": 1450, "grad_norm_max": 0.5792702436447144, "hidden_mse": 0.11123776510357858, "loss": 0.06667226417921483, "lr_blocks": 2.132117818244771e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 68.46298551559448}
|
| 147 |
+
{"avg_micro_time_s": 1.8742496912454953, "epoch": 22, "flow_mse": 0.05278900931589306, "global_step": 1460, "grad_norm_max": 0.41618072986602783, "hidden_mse": 0.10622980045154691, "loss": 0.06341198971495032, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 68.46298551559448}
|
| 148 |
+
{"avg_micro_time_s": 2.0268068581106493, "epoch": 22, "flow_mse": 0.05427227606996894, "global_step": 1470, "grad_norm_max": 0.38854727149009705, "hidden_mse": 0.10826945109292865, "loss": 0.06509922100231051, "lr_blocks": 1.990924884239758e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 68.46298551559448}
|
| 149 |
+
{"avg_micro_time_s": 1.6323719802472625, "epoch": 22, "flow_mse": 0.05504520414397121, "global_step": 1480, "grad_norm_max": 0.42108166217803955, "hidden_mse": 0.11108364090323448, "loss": 0.06615356872789561, "lr_blocks": 1.9216926233717087e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 68.46298551559448}
|
| 150 |
+
{"avg_micro_time_s": 1.7635257252361045, "epoch": 22, "flow_mse": 0.05449590594507754, "global_step": 1490, "grad_norm_max": 0.2980191111564636, "hidden_mse": 0.10923452377319337, "loss": 0.06541935848072171, "lr_blocks": 1.8533980447508138e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 68.46298551559448}
|
| 151 |
+
{"avg_micro_time_s": 1.7368722179773615, "epoch": 22, "flow_mse": 0.05059092580340803, "global_step": 1500, "grad_norm_max": 0.3388434052467346, "hidden_mse": 0.10435767890885472, "loss": 0.06102669364772737, "lr_blocks": 1.7860619515673034e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 68.46298551559448}
|
| 152 |
+
{"avg_micro_time_s": 2.083392089595145, "epoch": 22, "flow_mse": 0.05451004351489246, "global_step": 1510, "grad_norm_max": 0.31937962770462036, "hidden_mse": 0.10862588789314032, "loss": 0.0653726323042065, "lr_blocks": 1.7197048550474643e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 68.46298551559448}
|
| 153 |
+
{"avg_micro_time_s": 1.7850786256254652, "epoch": 23, "flow_mse": 0.05411349283531308, "global_step": 1520, "grad_norm_max": 0.43102946877479553, "hidden_mse": 0.1085999496281147, "loss": 0.06497348803095519, "lr_blocks": 1.6543469682057105e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 68.46298551559448}
|
| 154 |
+
{"avg_micro_time_s": 1.6930420300239348, "epoch": 23, "flow_mse": 0.05289990794844926, "global_step": 1530, "grad_norm_max": 0.3967125415802002, "hidden_mse": 0.10759261148050428, "loss": 0.06365916887298227, "lr_blocks": 1.5900081996875083e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 68.46298551559448}
|
| 155 |
+
{"avg_micro_time_s": 1.8721173314916086, "epoch": 23, "flow_mse": 0.05183863551355899, "global_step": 1540, "grad_norm_max": 0.65227872133255, "hidden_mse": 0.10619828049093485, "loss": 0.062458463944494724, "lr_blocks": 1.5267081477050132e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 68.46298551559448}
|
| 156 |
+
{"avg_micro_time_s": 1.9260684134074837, "epoch": 23, "flow_mse": 0.05517242061905563, "global_step": 1550, "grad_norm_max": 0.3173617720603943, "hidden_mse": 0.11037433873862029, "loss": 0.06620985479094088, "lr_blocks": 1.4644660940672628e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 68.46298551559448}
|
| 157 |
+
{"avg_micro_time_s": 2.00732960163441, "epoch": 23, "flow_mse": 0.052165593765676026, "global_step": 1560, "grad_norm_max": 0.48366814851760864, "hidden_mse": 0.10552244568243624, "loss": 0.06271783891133964, "lr_blocks": 1.4033009983067454e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 68.46298551559448}
|
| 158 |
+
{"avg_micro_time_s": 2.0028102155803937, "epoch": 23, "flow_mse": 0.054785998072475195, "global_step": 1570, "grad_norm_max": 0.32774579524993896, "hidden_mse": 0.1094754220917821, "loss": 0.0657335405703634, "lr_blocks": 1.3432314919041478e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 68.46298551559448}
|
| 159 |
+
{"avg_micro_time_s": 1.8898658655904002, "epoch": 23, "flow_mse": 0.05355181619524956, "global_step": 1580, "grad_norm_max": 0.29386377334594727, "hidden_mse": 0.10809278432279826, "loss": 0.06436109468340874, "lr_blocks": 1.2842758726130283e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 68.46298551559448}
|
| 160 |
+
{"avg_micro_time_s": 1.8055248979391765, "epoch": 24, "flow_mse": 0.05223282501101494, "global_step": 1590, "grad_norm_max": 0.30623888969421387, "hidden_mse": 0.10633350862190127, "loss": 0.06286617605946958, "lr_blocks": 1.22645209888614e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 68.46298551559448}
|
| 161 |
+
{"avg_micro_time_s": 1.697861864778679, "epoch": 24, "flow_mse": 0.05407982333563269, "global_step": 1600, "grad_norm_max": 0.41349685192108154, "hidden_mse": 0.1090579808689654, "loss": 0.06498562130145728, "lr_blocks": 1.1697777844051105e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 68.46298551559448}
|
| 162 |
+
{"avg_micro_time_s": 1.858836678104126, "epoch": 24, "flow_mse": 0.0515467718243599, "global_step": 1610, "grad_norm_max": 0.2459108531475067, "hidden_mse": 0.10587860029190779, "loss": 0.06213463232852519, "lr_blocks": 1.1142701927151456e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 68.46298551559448}
|
| 163 |
+
{"avg_micro_time_s": 1.7442898534587585, "epoch": 24, "flow_mse": 0.05539359417743981, "global_step": 1620, "grad_norm_max": 0.4176762104034424, "hidden_mse": 0.11120789768174291, "loss": 0.06651438400149345, "lr_blocks": 1.0599462319663906e-06, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 68.46298551559448}
|
| 164 |
+
{"avg_micro_time_s": 1.9312700220412808, "epoch": 24, "flow_mse": 0.05194714558310807, "global_step": 1630, "grad_norm_max": 0.27055931091308594, "hidden_mse": 0.10536178648471833, "loss": 0.06248332438990474, "lr_blocks": 1.006822449763537e-06, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 68.46298551559448}
|
| 165 |
+
{"avg_micro_time_s": 1.9848020562625608, "epoch": 24, "flow_mse": 0.05159918488934637, "global_step": 1640, "grad_norm_max": 0.2410484105348587, "hidden_mse": 0.10558492876589298, "loss": 0.06215767813846469, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 68.46298551559448}
|
| 166 |
+
{"avg_micro_time_s": 1.9862558376597008, "epoch": 24, "flow_mse": 0.051943618524819615, "global_step": 1650, "grad_norm_max": 0.19621193408966064, "hidden_mse": 0.10532341394573451, "loss": 0.062475959723815325, "lr_blocks": 9.042397785550405e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 68.46298551559448}
|
| 167 |
+
{"avg_micro_time_s": 1.90772508844384, "epoch": 25, "flow_mse": 0.05241604868788272, "global_step": 1660, "grad_norm_max": 0.24477998912334442, "hidden_mse": 0.10615330189466476, "loss": 0.06303137941285968, "lr_blocks": 8.54812137224792e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 68.46298551559448}
|
| 168 |
+
{"avg_micro_time_s": 2.0734705874245267, "epoch": 25, "flow_mse": 0.052382499817758804, "global_step": 1670, "grad_norm_max": 0.18695969879627228, "hidden_mse": 0.10536184627562761, "loss": 0.06291868472471833, "lr_blocks": 8.066471602728804e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 68.46298551559448}
|
| 169 |
+
{"avg_micro_time_s": 1.784898155926203, "epoch": 25, "flow_mse": 0.05266823512502015, "global_step": 1680, "grad_norm_max": 0.30488482117652893, "hidden_mse": 0.10819421242922544, "loss": 0.0634876563679427, "lr_blocks": 7.597595192178702e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 68.46298551559448}
|
| 170 |
+
{"avg_micro_time_s": 1.7039467210503063, "epoch": 25, "flow_mse": 0.053952160244807604, "global_step": 1690, "grad_norm_max": 0.21721898019313812, "hidden_mse": 0.10940732667222619, "loss": 0.06489289305172861, "lr_blocks": 7.141634964894389e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 68.46298551559448}
|
| 171 |
+
{"avg_micro_time_s": 1.7925323831252173, "epoch": 25, "flow_mse": 0.054602593369781974, "global_step": 1700, "grad_norm_max": 0.2606789171695709, "hidden_mse": 0.11060061100870372, "loss": 0.06566265453584492, "lr_blocks": 6.698729810778065e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 68.46298551559448}
|
| 172 |
+
{"avg_micro_time_s": 1.8506990019581282, "epoch": 25, "flow_mse": 0.05084327077493072, "global_step": 1710, "grad_norm_max": 0.18723498284816742, "hidden_mse": 0.10409033391624689, "loss": 0.06125230453908444, "lr_blocks": 6.269014643030214e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 68.46298551559448}
|
| 173 |
+
{"avg_micro_time_s": 1.696594527331763, "epoch": 26, "flow_mse": 0.04985265554860235, "global_step": 1720, "grad_norm_max": 0.16147896647453308, "hidden_mse": 0.10344723165035248, "loss": 0.060197379020974044, "lr_blocks": 5.852620357053651e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 68.46298551559448}
|
| 174 |
+
{"avg_micro_time_s": 1.7064527680224275, "epoch": 26, "flow_mse": 0.052878752723336225, "global_step": 1730, "grad_norm_max": 0.21210241317749023, "hidden_mse": 0.10743851037696005, "loss": 0.0636226034257561, "lr_blocks": 5.449673790581611e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 68.46298551559448}
|
| 175 |
+
{"avg_micro_time_s": 1.9300253918670933, "epoch": 26, "flow_mse": 0.05500989542342723, "global_step": 1740, "grad_norm_max": 0.18679021298885345, "hidden_mse": 0.11223258469253779, "loss": 0.06623315447941423, "lr_blocks": 5.06029768504166e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 68.46298551559448}
|
| 176 |
+
{"avg_micro_time_s": 1.6863267055538018, "epoch": 26, "flow_mse": 0.05216321973130107, "global_step": 1750, "grad_norm_max": 0.1739465296268463, "hidden_mse": 0.10685156406834723, "loss": 0.06284837638959288, "lr_blocks": 4.6846106481675035e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 68.46298551559448}
|
| 177 |
+
{"avg_micro_time_s": 1.9015966636099622, "epoch": 26, "flow_mse": 0.05238161133602262, "global_step": 1760, "grad_norm_max": 0.16062317788600922, "hidden_mse": 0.1064825899899006, "loss": 0.06302987090311944, "lr_blocks": 4.322727117869951e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 68.46298551559448}
|
| 178 |
+
{"avg_micro_time_s": 1.89506446889136, "epoch": 26, "flow_mse": 0.05142894261516631, "global_step": 1770, "grad_norm_max": 0.19871976971626282, "hidden_mse": 0.10540924789384008, "loss": 0.06196986767463386, "lr_blocks": 3.9747573273779816e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 68.46298551559448}
|
| 179 |
+
{"avg_micro_time_s": 2.140559560073598, "epoch": 26, "flow_mse": 0.052294114232063295, "global_step": 1780, "grad_norm_max": 0.13666212558746338, "hidden_mse": 0.10540376659482718, "loss": 0.06283449097536505, "lr_blocks": 3.6408072716606346e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 68.46298551559448}
|
| 180 |
+
{"avg_micro_time_s": 1.9869108784609126, "epoch": 27, "flow_mse": 0.052665405767038466, "global_step": 1790, "grad_norm_max": 0.1539895087480545, "hidden_mse": 0.1076177385635674, "loss": 0.06342717967927457, "lr_blocks": 3.320978675139919e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 68.46298551559448}
|
| 181 |
+
{"avg_micro_time_s": 1.8483484975920874, "epoch": 27, "flow_mse": 0.05281953318044544, "global_step": 1800, "grad_norm_max": 0.1717279851436615, "hidden_mse": 0.10747445961460472, "loss": 0.06356697916053236, "lr_blocks": 3.015368960704584e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 68.46298551559448}
|
| 182 |
+
{"avg_micro_time_s": 1.7725311637754204, "epoch": 27, "flow_mse": 0.053238363936543465, "global_step": 1810, "grad_norm_max": 0.129792258143425, "hidden_mse": 0.10810153055936099, "loss": 0.06404851726256311, "lr_blocks": 2.724071220034158e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 68.46298551559448}
|
| 183 |
+
{"avg_micro_time_s": 1.9046508581523085, "epoch": 27, "flow_mse": 0.05027513452805579, "global_step": 1820, "grad_norm_max": 0.13140571117401123, "hidden_mse": 0.1037200196646154, "loss": 0.06064713676460087, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 68.46298551559448}
|
| 184 |
+
{"avg_micro_time_s": 1.7696023706230335, "epoch": 27, "flow_mse": 0.053843728452920916, "global_step": 1830, "grad_norm_max": 0.10490740090608597, "hidden_mse": 0.10948641048744322, "loss": 0.06479236944578588, "lr_blocks": 2.1847622018482283e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 68.46298551559448}
|
| 185 |
+
{"avg_micro_time_s": 1.8017618429686992, "epoch": 27, "flow_mse": 0.05273311133496464, "global_step": 1840, "grad_norm_max": 0.1325991004705429, "hidden_mse": 0.10784135181456805, "loss": 0.06351724686101079, "lr_blocks": 1.9369152030840553e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 68.46298551559448}
|
| 186 |
+
{"avg_micro_time_s": 1.8663546495808987, "epoch": 28, "flow_mse": 0.05010635161306709, "global_step": 1850, "grad_norm_max": 0.13209085166454315, "hidden_mse": 0.10319266710430385, "loss": 0.06042561884969473, "lr_blocks": 1.7037086855465902e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 68.46298551559448}
|
| 187 |
+
{"avg_micro_time_s": 2.023102572160133, "epoch": 28, "flow_mse": 0.0542366745416075, "global_step": 1860, "grad_norm_max": 0.09557513147592545, "hidden_mse": 0.10798764936625958, "loss": 0.06503543937578797, "lr_blocks": 1.4852136862001766e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 68.46298551559448}
|
| 188 |
+
{"avg_micro_time_s": 1.9713299808965532, "epoch": 28, "flow_mse": 0.050487085804343225, "global_step": 1870, "grad_norm_max": 0.09084595739841461, "hidden_mse": 0.1046027010306716, "loss": 0.06094735627993941, "lr_blocks": 1.2814967607382433e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 68.46298551559448}
|
| 189 |
+
{"avg_micro_time_s": 1.765306223012158, "epoch": 28, "flow_mse": 0.052992650866508485, "global_step": 1880, "grad_norm_max": 0.1068035289645195, "hidden_mse": 0.10854229601100088, "loss": 0.06384688057005405, "lr_blocks": 1.0926199633097156e-07, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 68.46298551559448}
|
| 190 |
+
{"avg_micro_time_s": 1.8488479020117667, "epoch": 28, "flow_mse": 0.05185321508906782, "global_step": 1890, "grad_norm_max": 0.0908748134970665, "hidden_mse": 0.10693074595183134, "loss": 0.06254628994502127, "lr_blocks": 9.186408276168012e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 68.46298551559448}
|
| 191 |
+
{"avg_micro_time_s": 1.835118718787271, "epoch": 28, "flow_mse": 0.050304826558567585, "global_step": 1900, "grad_norm_max": 0.07472742348909378, "hidden_mse": 0.10459219636395574, "loss": 0.06076404661871493, "lr_blocks": 7.59612349389599e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 68.46298551559448}
|
| 192 |
+
{"avg_micro_time_s": 1.8683127601994785, "epoch": 28, "flow_mse": 0.05247240727767349, "global_step": 1910, "grad_norm_max": 0.0885327011346817, "hidden_mse": 0.1072016378864646, "loss": 0.06319257123395801, "lr_blocks": 6.15582970243117e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 68.46298551559448}
|
| 193 |
+
{"avg_micro_time_s": 1.6088743058397086, "epoch": 29, "flow_mse": 0.05225406056270004, "global_step": 1920, "grad_norm_max": 0.06508878618478775, "hidden_mse": 0.10751305781304837, "loss": 0.06300536664202809, "lr_blocks": 4.865965629214819e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 68.46298551559448}
|
| 194 |
+
{"avg_micro_time_s": 1.6788693653565134, "epoch": 29, "flow_mse": 0.05310070822015405, "global_step": 1930, "grad_norm_max": 0.09013048559427261, "hidden_mse": 0.10843602363020183, "loss": 0.0639443107880652, "lr_blocks": 3.726924179339009e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 68.46298551559448}
|
| 195 |
+
{"avg_micro_time_s": 1.8483045230590507, "epoch": 29, "flow_mse": 0.05225380118936301, "global_step": 1940, "grad_norm_max": 0.05455300584435463, "hidden_mse": 0.10687568383291364, "loss": 0.06294136978685856, "lr_blocks": 2.7390523158633552e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 68.46298551559448}
|
| 196 |
+
{"avg_micro_time_s": 2.07668071983353, "epoch": 29, "flow_mse": 0.052197744604200125, "global_step": 1950, "grad_norm_max": 0.06954554468393326, "hidden_mse": 0.10710220159962774, "loss": 0.06290796506218613, "lr_blocks": 1.9026509541272276e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 68.46298551559448}
|
| 197 |
+
{"avg_micro_time_s": 1.890728423825931, "epoch": 29, "flow_mse": 0.052322149462997916, "global_step": 1960, "grad_norm_max": 0.06443354487419128, "hidden_mse": 0.10821447782218457, "loss": 0.06314359758980573, "lr_blocks": 1.2179748700879013e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 68.46298551559448}
|
| 198 |
+
{"avg_micro_time_s": 1.7953088903203025, "epoch": 29, "flow_mse": 0.05093962624669075, "global_step": 1970, "grad_norm_max": 0.07561659812927246, "hidden_mse": 0.10430180076509715, "loss": 0.06136980666778982, "lr_blocks": 6.852326227130835e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 68.46298551559448}
|
| 199 |
+
{"avg_micro_time_s": 1.842160684411647, "epoch": 29, "flow_mse": 0.053311998723074794, "global_step": 1980, "grad_norm_max": 0.04818754270672798, "hidden_mse": 0.10754476888105274, "loss": 0.06406647614203394, "lr_blocks": 3.0458649045211897e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 68.46298551559448}
|
| 200 |
+
{"avg_micro_time_s": 1.7273795908869944, "epoch": 30, "flow_mse": 0.051652479777112605, "global_step": 1990, "grad_norm_max": 0.061520446091890335, "hidden_mse": 0.10536492271348835, "loss": 0.06218897206708789, "lr_blocks": 7.615242180436521e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 68.46298551559448}
|
| 201 |
+
{"avg_micro_time_s": 1.9548408255825054, "epoch": 30, "flow_mse": 0.05280004227533937, "global_step": 2000, "grad_norm_max": 0.054393548518419266, "hidden_mse": 0.10706793488934636, "loss": 0.06350683607161045, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 68.46298551559448}
|
checkpoints/predictor_v4_1000_seed0/training_and_sync.lock
ADDED
|
File without changes
|
checkpoints/predictor_v4_1000_seed0/training_and_sync.log
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-25T14:41:01Z] 从完整 NVMe 数据启动 8 GPU Predictor-v4 训练。
|
| 2 |
+
WARNING:__main__:
|
| 3 |
+
*****************************************
|
| 4 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 5 |
+
*****************************************
|
| 6 |
+
[rank0]:[W725 14:41:19.028815623 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 7 |
+
[rank4]:[W725 14:41:20.741616993 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 8 |
+
[rank1]:[W725 14:41:20.764551054 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 9 |
+
[rank3]:[W725 14:41:20.778364499 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 10 |
+
[rank5]:[W725 14:41:20.781348410 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 11 |
+
[rank6]:[W725 14:41:20.783009518 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 12 |
+
[rank7]:[W725 14:41:20.784113397 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 13 |
+
[rank2]:[W725 14:41:20.784201989 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 14 |
+
swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use
|
| 15 |
+
`swanlab login --relogin` to force relogin
|
| 16 |
+
swanlab: Tracking run with swanlab version 0.9.0
|
| 17 |
+
swanlab: 💾 Run data saved at
|
| 18 |
+
/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/
|
| 19 |
+
run-20260725_144159-4kxa9v7b
|
| 20 |
+
swanlab: 📁 View project at
|
| 21 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 22 |
+
swanlab: 🚀 View run pre...8-prompts1000-seed0-bs32x8-from-scratch at
|
| 23 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/4kxa9v7b
|
| 24 |
+
{"swanlab_run_id": "4kxa9v7b"}
|
| 25 |
+
{
|
| 26 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 27 |
+
"prompt_count": 1000,
|
| 28 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 29 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 30 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 31 |
+
"max_records": null,
|
| 32 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 33 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 34 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 35 |
+
"strict_teacher_load": true,
|
| 36 |
+
"source_block_ids": [
|
| 37 |
+
1,
|
| 38 |
+
28
|
| 39 |
+
],
|
| 40 |
+
"timestep_shift": 5.0,
|
| 41 |
+
"local_attn_size": -1,
|
| 42 |
+
"sink_size": 0,
|
| 43 |
+
"latent_height": 60,
|
| 44 |
+
"latent_width": 104,
|
| 45 |
+
"max_steps": 2000,
|
| 46 |
+
"batch_size": 32,
|
| 47 |
+
"gradient_accumulation_steps": 1,
|
| 48 |
+
"num_workers": 0,
|
| 49 |
+
"pin_memory": false,
|
| 50 |
+
"persistent_workers": false,
|
| 51 |
+
"fusion_lr": 0.0001,
|
| 52 |
+
"blocks_lr": 1e-05,
|
| 53 |
+
"beta1": 0.9,
|
| 54 |
+
"beta2": 0.95,
|
| 55 |
+
"weight_decay": 0.01,
|
| 56 |
+
"scheduler_warmup_steps": 100,
|
| 57 |
+
"block_warmup_steps": 100,
|
| 58 |
+
"hidden_loss_weight": 0.1,
|
| 59 |
+
"flow_loss_weight": 1.0,
|
| 60 |
+
"grad_clip": 1.0,
|
| 61 |
+
"fp32_trainable_params": true,
|
| 62 |
+
"gradient_checkpointing": false,
|
| 63 |
+
"find_unused_parameters": false,
|
| 64 |
+
"log_every": 10,
|
| 65 |
+
"save_every": 100,
|
| 66 |
+
"keep_snapshots": 20,
|
| 67 |
+
"resume": null,
|
| 68 |
+
"use_wandb": false,
|
| 69 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 70 |
+
"wandb_entity": null,
|
| 71 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 72 |
+
"use_swanlab": true,
|
| 73 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 74 |
+
"swanlab_workspace": null,
|
| 75 |
+
"swanlab_experiment": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs32x8-from-scratch",
|
| 76 |
+
"swanlab_description": "Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training",
|
| 77 |
+
"swanlab_tags": [
|
| 78 |
+
"predictor-v4",
|
| 79 |
+
"self-forcing",
|
| 80 |
+
"wan2.1-1.3b",
|
| 81 |
+
"offline-prefeature",
|
| 82 |
+
"blocks-1-28",
|
| 83 |
+
"8gpu",
|
| 84 |
+
"seed0"
|
| 85 |
+
],
|
| 86 |
+
"swanlab_mode": "cloud",
|
| 87 |
+
"supervision_pairs": [
|
| 88 |
+
[
|
| 89 |
+
0,
|
| 90 |
+
1
|
| 91 |
+
],
|
| 92 |
+
[
|
| 93 |
+
1,
|
| 94 |
+
2
|
| 95 |
+
],
|
| 96 |
+
[
|
| 97 |
+
2,
|
| 98 |
+
3
|
| 99 |
+
]
|
| 100 |
+
],
|
| 101 |
+
"manifest_records": 6000,
|
| 102 |
+
"dataset_pairs": 18000,
|
| 103 |
+
"steps_per_epoch_per_rank": 66,
|
| 104 |
+
"world_size": 8,
|
| 105 |
+
"global_batch_size": 256,
|
| 106 |
+
"trainable_parameters": 104684032,
|
| 107 |
+
"parameter_groups": {
|
| 108 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 109 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 110 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 111 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 112 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 113 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 114 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 115 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 116 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 117 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 118 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 119 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 120 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 121 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 122 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 123 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 124 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 125 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 126 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 127 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 128 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 129 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 130 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 131 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 132 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 133 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 134 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 135 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 136 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 137 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 138 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 139 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 140 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 141 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 142 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 143 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 144 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 145 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 146 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 147 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 148 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 149 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 150 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 151 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 152 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 153 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 154 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 155 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 156 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 157 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 158 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 159 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 160 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 161 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 162 |
+
"residual_out.weight": "fusion_residual",
|
| 163 |
+
"residual_out.bias": "fusion_residual"
|
| 164 |
+
},
|
| 165 |
+
"activation_dtype": "bfloat16",
|
| 166 |
+
"trainable_parameter_dtype": "float32",
|
| 167 |
+
"swanlab_run_id": "4kxa9v7b"
|
| 168 |
+
}
|
| 169 |
+
{"avg_micro_time_s": 7.782912929877057, "epoch": 0, "flow_mse": 1.728674739599228, "global_step": 1, "grad_norm_max": 1.224261999130249, "hidden_mse": 0.34973957017064095, "loss": 1.7636487036943436, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 59.944087982177734}
|
| 170 |
+
{"avg_micro_time_s": 1.852714978063078, "epoch": 0, "flow_mse": 1.7325291699833338, "global_step": 10, "grad_norm_max": 1.5914783477783203, "hidden_mse": 0.36041637013355887, "loss": 1.7685708122120962, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 68.46298551559448}
|
| 171 |
+
{"avg_micro_time_s": 1.846399891503097, "epoch": 0, "flow_mse": 1.699481675028801, "global_step": 20, "grad_norm_max": 1.9605422019958496, "hidden_mse": 0.38648081049323085, "loss": 1.7381297543644907, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 172 |
+
{"avg_micro_time_s": 2.076498315717618, "epoch": 0, "flow_mse": 1.5910302087664605, "global_step": 30, "grad_norm_max": 1.6393431425094604, "hidden_mse": 0.6486873880028725, "loss": 1.6558989509940147, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 173 |
+
{"avg_micro_time_s": 1.8261107150639875, "epoch": 0, "flow_mse": 1.4074327528476716, "global_step": 40, "grad_norm_max": 1.6300629377365112, "hidden_mse": 1.5101009473204614, "loss": 1.5584428489208222, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 174 |
+
{"avg_micro_time_s": 1.7557343540451258, "epoch": 0, "flow_mse": 1.3375679016113282, "global_step": 50, "grad_norm_max": 1.179707407951355, "hidden_mse": 1.7423540219664575, "loss": 1.5118033051490785, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 68.46298551559448}
|
| 175 |
+
{"avg_micro_time_s": 1.9954431485108217, "epoch": 0, "flow_mse": 1.288495972752571, "global_step": 60, "grad_norm_max": 0.9528158903121948, "hidden_mse": 1.6803042054176331, "loss": 1.4565263986587524, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 176 |
+
{"avg_micro_time_s": 2.073848069201631, "epoch": 1, "flow_mse": 1.2329863071441651, "global_step": 70, "grad_norm_max": 2.1396095752716064, "hidden_mse": 1.468385949730873, "loss": 1.3798249036073686, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 177 |
+
{"avg_micro_time_s": 1.9025867714255584, "epoch": 1, "flow_mse": 1.1653944417834283, "global_step": 80, "grad_norm_max": 3.283475875854492, "hidden_mse": 1.3008124545216562, "loss": 1.2954756870865822, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 178 |
+
{"avg_micro_time_s": 1.8846276455253248, "epoch": 1, "flow_mse": 1.0928176268935204, "global_step": 90, "grad_norm_max": 2.976677179336548, "hidden_mse": 1.1887190774083138, "loss": 1.211689528822899, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 68.46298551559448}
|
| 179 |
+
{"avg_micro_time_s": 2.0524098566529574, "epoch": 1, "flow_mse": 1.0091206416487695, "global_step": 100, "grad_norm_max": 2.0720155239105225, "hidden_mse": 1.1053067579865457, "loss": 1.119651308655739, "lr_blocks": 1.0000000000000001e-07, "lr_fusion": 0.0001, "peak_memory_gib": 68.46298551559448}
|
| 180 |
+
{"avg_micro_time_s": 1.8157492470301806, "epoch": 1, "flow_mse": 0.9281245060265064, "global_step": 110, "grad_norm_max": 3.5477075576782227, "hidden_mse": 1.0355910658836365, "loss": 1.031683611869812, "lr_blocks": 1.1e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 68.46298551559448}
|
| 181 |
+
{"avg_micro_time_s": 1.7559494119937882, "epoch": 1, "flow_mse": 0.8380454987287522, "global_step": 120, "grad_norm_max": 3.569852828979492, "hidden_mse": 1.0150623381137849, "loss": 0.9395517356693746, "lr_blocks": 2.1000000000000002e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 68.46298551559448}
|
| 182 |
+
{"avg_micro_time_s": 1.9936403229308781, "epoch": 1, "flow_mse": 0.7462111793458462, "global_step": 130, "grad_norm_max": 2.9960243701934814, "hidden_mse": 0.9869170427322388, "loss": 0.8449028812348843, "lr_blocks": 3.1000000000000004e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 68.46298551559448}
|
| 183 |
+
{"avg_micro_time_s": 1.8761029311470339, "epoch": 2, "flow_mse": 0.6552131280303002, "global_step": 140, "grad_norm_max": 3.033362627029419, "hidden_mse": 0.9761701941490174, "loss": 0.7528301522135735, "lr_blocks": 4.1e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 68.46298551559448}
|
| 184 |
+
{"avg_micro_time_s": 1.6636652560366203, "epoch": 2, "flow_mse": 0.5641707576811313, "global_step": 150, "grad_norm_max": 3.3895790576934814, "hidden_mse": 0.9558152981102467, "loss": 0.6597522847354412, "lr_blocks": 5.1e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 68.46298551559448}
|
| 185 |
+
{"avg_micro_time_s": 1.8629546527619825, "epoch": 2, "flow_mse": 0.49576797895133495, "global_step": 160, "grad_norm_max": 4.9101152420043945, "hidden_mse": 0.9376181185245515, "loss": 0.5895297914743424, "lr_blocks": 6.1e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 68.46298551559448}
|
| 186 |
+
{"avg_micro_time_s": 1.865570171797299, "epoch": 2, "flow_mse": 0.44515644647181035, "global_step": 170, "grad_norm_max": 3.2292189598083496, "hidden_mse": 0.9383152291178704, "loss": 0.5389879673719407, "lr_blocks": 7.100000000000001e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 68.46298551559448}
|
| 187 |
+
{"avg_micro_time_s": 2.0450479036619074, "epoch": 2, "flow_mse": 0.38247467912733557, "global_step": 180, "grad_norm_max": 2.7824766635894775, "hidden_mse": 0.9554134920239449, "loss": 0.47801603078842164, "lr_blocks": 8.1e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 68.46298551559448}
|
| 188 |
+
{"avg_micro_time_s": 2.016261681757169, "epoch": 2, "flow_mse": 0.3393749196082354, "global_step": 190, "grad_norm_max": 2.3986711502075195, "hidden_mse": 0.9583184562623501, "loss": 0.4352067667990923, "lr_blocks": 9.100000000000001e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 68.46298551559448}
|
| 189 |
+
{"avg_micro_time_s": 1.955125979850709, "epoch": 3, "flow_mse": 0.2995503291487694, "global_step": 200, "grad_norm_max": 2.834411144256592, "hidden_mse": 0.9311102323234082, "loss": 0.39266135394573215, "lr_blocks": 1e-05, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 68.46298551559448}
|
| 190 |
+
{"avg_micro_time_s": 1.9450525438660407, "epoch": 3, "flow_mse": 0.26215458251535895, "global_step": 210, "grad_norm_max": 2.5127832889556885, "hidden_mse": 0.9175012007355691, "loss": 0.35390470623970033, "lr_blocks": 9.999238475781957e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 68.46298551559448}
|
| 191 |
+
{"avg_micro_time_s": 1.794988799588464, "epoch": 3, "flow_mse": 0.24451670069247486, "global_step": 220, "grad_norm_max": 3.86234712600708, "hidden_mse": 0.8481401547789574, "loss": 0.32933071814477444, "lr_blocks": 9.99695413509548e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 68.46298551559448}
|
| 192 |
+
{"avg_micro_time_s": 1.821383213158697, "epoch": 3, "flow_mse": 0.21927626319229604, "global_step": 230, "grad_norm_max": 2.894408702850342, "hidden_mse": 0.7950924828648568, "loss": 0.2987855151295662, "lr_blocks": 9.993147673772869e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 68.46298551559448}
|
| 193 |
+
{"avg_micro_time_s": 1.7578791950494634, "epoch": 3, "flow_mse": 0.21038782224059105, "global_step": 240, "grad_norm_max": 4.587767124176025, "hidden_mse": 0.7239097714424134, "loss": 0.2827788006514311, "lr_blocks": 9.987820251299121e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 68.46298551559448}
|
| 194 |
+
{"avg_micro_time_s": 1.9880367784600823, "epoch": 3, "flow_mse": 0.18143612183630467, "global_step": 250, "grad_norm_max": 2.522402048110962, "hidden_mse": 0.6568384729325771, "loss": 0.24711996987462045, "lr_blocks": 9.980973490458728e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 68.46298551559448}
|
| 195 |
+
{"avg_micro_time_s": 2.1574960700992962, "epoch": 3, "flow_mse": 0.17069688346236944, "global_step": 260, "grad_norm_max": 2.6453685760498047, "hidden_mse": 0.5946614742279053, "loss": 0.23016303274780514, "lr_blocks": 9.972609476841368e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 68.46298551559448}
|
| 196 |
+
{"avg_micro_time_s": 1.7216765362391016, "epoch": 4, "flow_mse": 0.16511701364070178, "global_step": 270, "grad_norm_max": 4.101485252380371, "hidden_mse": 0.5324683826416731, "loss": 0.21836385373026135, "lr_blocks": 9.962730758206612e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 68.46298551559448}
|
| 197 |
+
{"avg_micro_time_s": 2.036255760942004, "epoch": 4, "flow_mse": 0.15243881475180387, "global_step": 280, "grad_norm_max": 3.777620792388916, "hidden_mse": 0.49607212990522387, "loss": 0.20204602926969528, "lr_blocks": 9.951340343707852e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 68.46298551559448}
|
| 198 |
+
2026-07-25 15:13:30.859 | ERROR |
|
| 199 |
+
swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST
|
| 200 |
+
https://api.swanlab.cn/api/house/metrics -> 522 (5106ms) trace:unknown | [ERR]
|
| 201 |
+
code=unknown code message=unknown error
|
| 202 |
+
2026-07-25 15:13:33.726 | WARNING |
|
| 203 |
+
swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due
|
| 204 |
+
to network or server issues, retrying automatically.
|
| 205 |
+
swanlab: Upload recovered, resuming normally.
|
| 206 |
+
{"avg_micro_time_s": 2.026448777728365, "epoch": 4, "flow_mse": 0.13280713185667992, "global_step": 290, "grad_norm_max": 2.864901542663574, "hidden_mse": 0.45720995105803014, "loss": 0.1785281287506223, "lr_blocks": 9.938441702975689e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 68.46298551559448}
|
| 207 |
+
{"avg_micro_time_s": 1.7816235546153623, "epoch": 4, "flow_mse": 0.1316212204284966, "global_step": 300, "grad_norm_max": 3.557399272918701, "hidden_mse": 0.4236030507832766, "loss": 0.17398152723908425, "lr_blocks": 9.924038765061042e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 68.46298551559448}
|
| 208 |
+
{"avg_micro_time_s": 1.7509075272682821, "epoch": 4, "flow_mse": 0.12680668467655778, "global_step": 310, "grad_norm_max": 2.9080326557159424, "hidden_mse": 0.3970677562057972, "loss": 0.16651346255093813, "lr_blocks": 9.908135917238321e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 68.46298551559448}
|
| 209 |
+
{"avg_micro_time_s": 1.94211900010705, "epoch": 4, "flow_mse": 0.1265168657526374, "global_step": 320, "grad_norm_max": 3.0853631496429443, "hidden_mse": 0.3776782535016537, "loss": 0.16428469233214857, "lr_blocks": 9.890738003669029e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 68.46298551559448}
|
| 210 |
+
{"avg_micro_time_s": 1.9201412342023105, "epoch": 4, "flow_mse": 0.11589663326740265, "global_step": 330, "grad_norm_max": 3.0462610721588135, "hidden_mse": 0.3677100505679846, "loss": 0.1526676405221224, "lr_blocks": 9.871850323926178e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 68.46298551559448}
|
| 211 |
+
{"avg_micro_time_s": 1.8802110167060164, "epoch": 5, "flow_mse": 0.10986252082511783, "global_step": 340, "grad_norm_max": 2.5844221115112305, "hidden_mse": 0.3474765542894602, "loss": 0.14461017716675997, "lr_blocks": 9.851478631379982e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 68.46298551559448}
|
| 212 |
+
{"avg_micro_time_s": 1.970779871637933, "epoch": 5, "flow_mse": 0.10572945503517986, "global_step": 350, "grad_norm_max": 2.6753783226013184, "hidden_mse": 0.33289663605391984, "loss": 0.13901912048459053, "lr_blocks": 9.829629131445342e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 68.46298551559448}
|
| 213 |
+
{"avg_micro_time_s": 1.8511785630384112, "epoch": 5, "flow_mse": 0.09914825893938542, "global_step": 360, "grad_norm_max": 2.757209539413452, "hidden_mse": 0.3192139830440283, "loss": 0.13106965916231275, "lr_blocks": 9.806308479691595e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 68.46298551559448}
|
| 214 |
+
{"avg_micro_time_s": 1.9199401149497135, "epoch": 5, "flow_mse": 0.0996528490446508, "global_step": 370, "grad_norm_max": 2.9345085620880127, "hidden_mse": 0.3086499966681004, "loss": 0.1305178500711918, "lr_blocks": 9.781523779815178e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 68.46298551559448}
|
| 215 |
+
{"avg_micro_time_s": 1.7542048408882693, "epoch": 5, "flow_mse": 0.11152176447212697, "global_step": 380, "grad_norm_max": 3.106746196746826, "hidden_mse": 0.30627303943037987, "loss": 0.1421490689739585, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 68.46298551559448}
|
| 216 |
+
{"avg_micro_time_s": 1.9700474178520382, "epoch": 5, "flow_mse": 0.10044981110841036, "global_step": 390, "grad_norm_max": 2.560649871826172, "hidden_mse": 0.2952960841357708, "loss": 0.1299794196151197, "lr_blocks": 9.727592877996585e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 68.46298551559448}
|
| 217 |
+
{"avg_micro_time_s": 1.6663271824698314, "epoch": 6, "flow_mse": 0.09705260461196304, "global_step": 400, "grad_norm_max": 2.532449722290039, "hidden_mse": 0.29162609074264767, "loss": 0.1262152137234807, "lr_blocks": 9.698463103929542e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 68.46298551559448}
|
| 218 |
+
{"avg_micro_time_s": 1.904068420661497, "epoch": 6, "flow_mse": 0.09287562295794488, "global_step": 410, "grad_norm_max": 2.16715145111084, "hidden_mse": 0.28314792811870576, "loss": 0.121190416906029, "lr_blocks": 9.667902132486009e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 68.46298551559448}
|
| 219 |
+
{"avg_micro_time_s": 1.744727985771897, "epoch": 6, "flow_mse": 0.09119896441698075, "global_step": 420, "grad_norm_max": 2.299543619155884, "hidden_mse": 0.275063331797719, "loss": 0.1187052981927991, "lr_blocks": 9.635919272833938e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 68.46298551559448}
|
| 220 |
+
{"avg_micro_time_s": 1.7956668802158675, "epoch": 6, "flow_mse": 0.08759928597137333, "global_step": 430, "grad_norm_max": 2.317425489425659, "hidden_mse": 0.2642714731395245, "loss": 0.11402643444016576, "lr_blocks": 9.602524267262202e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 68.46298551559448}
|
| 221 |
+
{"avg_micro_time_s": 1.9157415576424683, "epoch": 6, "flow_mse": 0.08734634332358837, "global_step": 440, "grad_norm_max": 2.760629415512085, "hidden_mse": 0.25739369336515666, "loss": 0.11308571305125953, "lr_blocks": 9.567727288213005e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 68.46298551559448}
|
| 222 |
+
{"avg_micro_time_s": 1.8787613131338732, "epoch": 6, "flow_mse": 0.08993547437712551, "global_step": 450, "grad_norm_max": 3.014104127883911, "hidden_mse": 0.25076083578169345, "loss": 0.11501155886799097, "lr_blocks": 9.531538935183252e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 68.46298551559448}
|
| 223 |
+
{"avg_micro_time_s": 1.8499236255913276, "epoch": 6, "flow_mse": 0.0833338218741119, "global_step": 460, "grad_norm_max": 2.0357143878936768, "hidden_mse": 0.24471132159233094, "loss": 0.10780495461076499, "lr_blocks": 9.493970231495836e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 68.46298551559448}
|
| 224 |
+
{"avg_micro_time_s": 1.8277902282425202, "epoch": 7, "flow_mse": 0.08892108229920269, "global_step": 470, "grad_norm_max": 2.870374917984009, "hidden_mse": 0.24110151771456004, "loss": 0.11303123468533159, "lr_blocks": 9.45503262094184e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 68.46298551559448}
|
| 225 |
+
{"avg_micro_time_s": 1.8054274757610984, "epoch": 7, "flow_mse": 0.08921196823939681, "global_step": 480, "grad_norm_max": 2.4050748348236084, "hidden_mse": 0.24109693542122843, "loss": 0.113321662414819, "lr_blocks": 9.414737964294636e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 226 |
+
{"avg_micro_time_s": 1.969536912314652, "epoch": 7, "flow_mse": 0.08481648499146105, "global_step": 490, "grad_norm_max": 4.0341668128967285, "hidden_mse": 0.23553607985377312, "loss": 0.10837009344249965, "lr_blocks": 9.37309853569698e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 68.46298551559448}
|
| 227 |
+
{"avg_micro_time_s": 1.7191957566945348, "epoch": 7, "flow_mse": 0.08171517765149475, "global_step": 500, "grad_norm_max": 2.7221522331237793, "hidden_mse": 0.22536225505173207, "loss": 0.10425140429288149, "lr_blocks": 9.330127018922195e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 68.46298551559448}
|
| 228 |
+
{"avg_micro_time_s": 1.8753900891068043, "epoch": 7, "flow_mse": 0.08210981907323003, "global_step": 510, "grad_norm_max": 1.9840606451034546, "hidden_mse": 0.22030116170644762, "loss": 0.10413993652909995, "lr_blocks": 9.285836503510562e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 68.46298551559448}
|
| 229 |
+
{"avg_micro_time_s": 1.9803101174184121, "epoch": 7, "flow_mse": 0.08212232678197325, "global_step": 520, "grad_norm_max": 2.06720232963562, "hidden_mse": 0.21767251621931794, "loss": 0.10388957895338535, "lr_blocks": 9.24024048078213e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 68.46298551559448}
|
| 230 |
+
{"avg_micro_time_s": 1.7816847848371253, "epoch": 8, "flow_mse": 0.07924596662633121, "global_step": 530, "grad_norm_max": 2.3816592693328857, "hidden_mse": 0.21423672661185267, "loss": 0.10066964011639357, "lr_blocks": 9.193352839727122e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 68.46298551559448}
|
| 231 |
+
{"avg_micro_time_s": 1.862245862412965, "epoch": 8, "flow_mse": 0.07841471419669688, "global_step": 540, "grad_norm_max": 2.3286426067352295, "hidden_mse": 0.2106100643053651, "loss": 0.09947572173550726, "lr_blocks": 9.145187862775208e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 68.46298551559448}
|
| 232 |
+
{"avg_micro_time_s": 2.0419705960011925, "epoch": 8, "flow_mse": 0.07392775514163077, "global_step": 550, "grad_norm_max": 1.3650901317596436, "hidden_mse": 0.2003183823078871, "loss": 0.09395959414541721, "lr_blocks": 9.09576022144496e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 68.46298551559448}
|
| 233 |
+
{"avg_micro_time_s": 1.6736296482136823, "epoch": 8, "flow_mse": 0.07821477656252683, "global_step": 560, "grad_norm_max": 2.5323493480682373, "hidden_mse": 0.19945396259427073, "loss": 0.09816017346456647, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 68.46298551559448}
|
| 234 |
+
{"avg_micro_time_s": 2.003066141484305, "epoch": 8, "flow_mse": 0.08001610483042897, "global_step": 570, "grad_norm_max": 2.0001752376556396, "hidden_mse": 0.19836203493177892, "loss": 0.09985230900347233, "lr_blocks": 8.993177550236464e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 68.46298551559448}
|
| 235 |
+
{"avg_micro_time_s": 1.9908761504004362, "epoch": 8, "flow_mse": 0.07824829332530499, "global_step": 580, "grad_norm_max": 2.1289918422698975, "hidden_mse": 0.19441615901887419, "loss": 0.09768990930169821, "lr_blocks": 8.94005376803361e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 68.46298551559448}
|
| 236 |
+
{"avg_micro_time_s": 1.722032713617955, "epoch": 8, "flow_mse": 0.07608461757190527, "global_step": 590, "grad_norm_max": 2.0547220706939697, "hidden_mse": 0.19045738708227875, "loss": 0.09513035686686636, "lr_blocks": 8.885729807284855e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 68.46298551559448}
|
| 237 |
+
{"avg_micro_time_s": 1.5744449160105434, "epoch": 9, "flow_mse": 0.07611198453232647, "global_step": 600, "grad_norm_max": 1.646276831626892, "hidden_mse": 0.1878601059317589, "loss": 0.09489799588918686, "lr_blocks": 8.83022221559489e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 68.46298551559448}
|
| 238 |
+
{"avg_micro_time_s": 1.9617489955402563, "epoch": 9, "flow_mse": 0.07234360408037901, "global_step": 610, "grad_norm_max": 2.1270275115966797, "hidden_mse": 0.18017836939543486, "loss": 0.09036144157871605, "lr_blocks": 8.773547901113862e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 68.46298551559448}
|
| 239 |
+
{"avg_micro_time_s": 1.899630580819212, "epoch": 9, "flow_mse": 0.07547258702106774, "global_step": 620, "grad_norm_max": 2.535357713699341, "hidden_mse": 0.17847571074962618, "loss": 0.09332015877589583, "lr_blocks": 8.715724127386971e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 68.46298551559448}
|
| 240 |
+
{"avg_micro_time_s": 1.9009491362783593, "epoch": 9, "flow_mse": 0.07615209170617164, "global_step": 630, "grad_norm_max": 2.885998487472534, "hidden_mse": 0.17413493599742652, "loss": 0.09356558537110687, "lr_blocks": 8.656768508095853e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 68.46298551559448}
|
| 241 |
+
{"avg_micro_time_s": 2.033017601488973, "epoch": 9, "flow_mse": 0.07332893046550452, "global_step": 640, "grad_norm_max": 1.3615809679031372, "hidden_mse": 0.17425470147281885, "loss": 0.09075440168380738, "lr_blocks": 8.596699001693257e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 68.46298551559448}
|
| 242 |
+
{"avg_micro_time_s": 1.6646606730137137, "epoch": 9, "flow_mse": 0.07165623712353408, "global_step": 650, "grad_norm_max": 2.018874406814575, "hidden_mse": 0.17005239613354206, "loss": 0.08866147724911572, "lr_blocks": 8.535533905932739e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 68.46298551559448}
|
| 243 |
+
{"avg_micro_time_s": 2.01382767639152, "epoch": 9, "flow_mse": 0.07549470965750515, "global_step": 660, "grad_norm_max": 1.9090533256530762, "hidden_mse": 0.1704067310318351, "loss": 0.0925353833474219, "lr_blocks": 8.473291852294986e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 68.46298551559448}
|
| 244 |
+
{"avg_micro_time_s": 2.044479904185573, "epoch": 10, "flow_mse": 0.07384380707517267, "global_step": 670, "grad_norm_max": 1.6318471431732178, "hidden_mse": 0.16633677743375302, "loss": 0.09047748530283571, "lr_blocks": 8.409991800312493e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 68.46298551559448}
|
| 245 |
+
{"avg_micro_time_s": 1.994344039564021, "epoch": 10, "flow_mse": 0.07081504138186574, "global_step": 680, "grad_norm_max": 2.8086788654327393, "hidden_mse": 0.16047870106995107, "loss": 0.08686291193589568, "lr_blocks": 8.345653031794292e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 68.46298551559448}
|
| 246 |
+
{"avg_micro_time_s": 1.7797691442654469, "epoch": 10, "flow_mse": 0.07098656282760203, "global_step": 690, "grad_norm_max": 1.6218206882476807, "hidden_mse": 0.15934279719367625, "loss": 0.086920843180269, "lr_blocks": 8.280295144952537e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 68.46298551559448}
|
| 247 |
+
{"avg_micro_time_s": 1.8434631714044372, "epoch": 10, "flow_mse": 0.07692104973830283, "global_step": 700, "grad_norm_max": 1.9783955812454224, "hidden_mse": 0.16042201202362777, "loss": 0.0929632511921227, "lr_blocks": 8.213938048432697e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 68.46298551559448}
|
| 248 |
+
{"avg_micro_time_s": 1.8933369300531924, "epoch": 10, "flow_mse": 0.06859853272326291, "global_step": 710, "grad_norm_max": 1.2764592170715332, "hidden_mse": 0.15258000474423172, "loss": 0.08385653356090189, "lr_blocks": 8.146601955249187e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 68.46298551559448}
|
| 249 |
+
{"avg_micro_time_s": 1.863453762442805, "epoch": 10, "flow_mse": 0.0677291950210929, "global_step": 720, "grad_norm_max": 2.756103992462158, "hidden_mse": 0.15030786208808422, "loss": 0.08275998155586423, "lr_blocks": 8.078307376628292e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 68.46298551559448}
|
| 250 |
+
{"avg_micro_time_s": 1.5886019194789698, "epoch": 11, "flow_mse": 0.06930862842127682, "global_step": 730, "grad_norm_max": 2.032028913497925, "hidden_mse": 0.14793342733755707, "loss": 0.08410197151824833, "lr_blocks": 8.009075115760243e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 68.46298551559448}
|
| 251 |
+
{"avg_micro_time_s": 1.7562155689141947, "epoch": 11, "flow_mse": 0.0670456328894943, "global_step": 740, "grad_norm_max": 1.5815403461456299, "hidden_mse": 0.14644069485366346, "loss": 0.08168970262631775, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 68.46298551559448}
|
| 252 |
+
{"avg_micro_time_s": 1.9935554676223548, "epoch": 11, "flow_mse": 0.06759789921343327, "global_step": 750, "grad_norm_max": 1.5702470541000366, "hidden_mse": 0.1456328984349966, "loss": 0.08216118961572648, "lr_blocks": 7.86788218175523e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 68.46298551559448}
|
| 253 |
+
{"avg_micro_time_s": 1.7392390850000083, "epoch": 11, "flow_mse": 0.0679298588540405, "global_step": 760, "grad_norm_max": 1.7798969745635986, "hidden_mse": 0.145010856539011, "loss": 0.0824309445451945, "lr_blocks": 7.795964517353734e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 68.46298551559448}
|
| 254 |
+
{"avg_micro_time_s": 1.883487955624878, "epoch": 11, "flow_mse": 0.06899964096955956, "global_step": 770, "grad_norm_max": 2.0383424758911133, "hidden_mse": 0.14355416363105178, "loss": 0.08335505775175989, "lr_blocks": 7.723195175075136e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 68.46298551559448}
|
| 255 |
+
{"avg_micro_time_s": 1.902795129803417, "epoch": 11, "flow_mse": 0.06839722348377109, "global_step": 780, "grad_norm_max": 1.341465950012207, "hidden_mse": 0.14199498109519482, "loss": 0.08259672177955509, "lr_blocks": 7.649596321166024e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 68.46298551559448}
|
| 256 |
+
{"avg_micro_time_s": 1.899291706715303, "epoch": 11, "flow_mse": 0.06829775697551668, "global_step": 790, "grad_norm_max": 1.6974772214889526, "hidden_mse": 0.14167611030861735, "loss": 0.08246536822989584, "lr_blocks": 7.575190374550272e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 68.46298551559448}
|
| 257 |
+
{"avg_micro_time_s": 1.9210877921635985, "epoch": 12, "flow_mse": 0.06724138637073339, "global_step": 800, "grad_norm_max": 1.6519050598144531, "hidden_mse": 0.13736099740490318, "loss": 0.08097748612053693, "lr_blocks": 7.500000000000001e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 68.46298551559448}
|
| 258 |
+
{"avg_micro_time_s": 1.839930806370103, "epoch": 12, "flow_mse": 0.06486202948726714, "global_step": 810, "grad_norm_max": 1.9736137390136719, "hidden_mse": 0.13331869533285498, "loss": 0.07819389933720232, "lr_blocks": 7.424048101231687e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 68.46298551559448}
|
| 259 |
+
{"avg_micro_time_s": 1.7203275968320668, "epoch": 12, "flow_mse": 0.06997489500790835, "global_step": 820, "grad_norm_max": 1.8273866176605225, "hidden_mse": 0.1383979432284832, "loss": 0.0838146897032857, "lr_blocks": 7.347357813929455e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 68.46298551559448}
|
| 260 |
+
{"avg_micro_time_s": 1.8308625894540456, "epoch": 12, "flow_mse": 0.06103807487525046, "global_step": 830, "grad_norm_max": 1.24065363407135, "hidden_mse": 0.12885685674846173, "loss": 0.07392376055940986, "lr_blocks": 7.269952498697734e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 68.46298551559448}
|
| 261 |
+
{"avg_micro_time_s": 1.951346460581408, "epoch": 12, "flow_mse": 0.0672075655311346, "global_step": 840, "grad_norm_max": 1.4299647808074951, "hidden_mse": 0.1341157573275268, "loss": 0.08061914164572954, "lr_blocks": 7.191855733945388e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 68.46298551559448}
|
| 262 |
+
{"avg_micro_time_s": 1.9852554764482193, "epoch": 12, "flow_mse": 0.06508104703389109, "global_step": 850, "grad_norm_max": 1.3047852516174316, "hidden_mse": 0.13088483419269323, "loss": 0.07816953058354557, "lr_blocks": 7.113091308703498e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 68.46298551559448}
|
| 263 |
+
{"avg_micro_time_s": 1.8463190050082632, "epoch": 13, "flow_mse": 0.06611472489312291, "global_step": 860, "grad_norm_max": 1.3949657678604126, "hidden_mse": 0.13323927726596593, "loss": 0.07943865223787726, "lr_blocks": 7.033683215379002e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 68.46298551559448}
|
| 264 |
+
{"avg_micro_time_s": 1.8538904657267268, "epoch": 13, "flow_mse": 0.0621907118242234, "global_step": 870, "grad_norm_max": 1.0381734371185303, "hidden_mse": 0.1276529028080404, "loss": 0.07495600231923163, "lr_blocks": 6.953655642446368e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 68.46298551559448}
|
| 265 |
+
{"avg_micro_time_s": 2.0335181808739438, "epoch": 13, "flow_mse": 0.06287266602739693, "global_step": 880, "grad_norm_max": 1.0403690338134766, "hidden_mse": 0.1264244217425585, "loss": 0.07551510813646019, "lr_blocks": 6.873032967079562e-06, "lr_fusion": 6.387014543809223e-05, "peak_memory_gib": 68.46298551559448}
|
| 266 |
+
{"avg_micro_time_s": 1.6593307598377578, "epoch": 13, "flow_mse": 0.0616169007960707, "global_step": 890, "grad_norm_max": 1.445999264717102, "hidden_mse": 0.12434439333155752, "loss": 0.07405133983120323, "lr_blocks": 6.7918397477265e-06, "lr_fusion": 6.307399704769099e-05, "peak_memory_gib": 68.46298551559448}
|
| 267 |
+
{"avg_micro_time_s": 1.6712814146303572, "epoch": 13, "flow_mse": 0.0653102396056056, "global_step": 900, "grad_norm_max": 2.982330322265625, "hidden_mse": 0.12486500488594175, "loss": 0.07779674059711397, "lr_blocks": 6.710100716628345e-06, "lr_fusion": 6.227427435703997e-05, "peak_memory_gib": 68.46298551559448}
|
| 268 |
+
{"avg_micro_time_s": 1.9577245332402526, "epoch": 13, "flow_mse": 0.06775598619133234, "global_step": 910, "grad_norm_max": 1.3592380285263062, "hidden_mse": 0.1261634855531156, "loss": 0.08037233473733069, "lr_blocks": 6.627840772285784e-06, "lr_fusion": 6.147119600233758e-05, "peak_memory_gib": 68.46298551559448}
|
| 269 |
+
{"avg_micro_time_s": 1.87712880110048, "epoch": 13, "flow_mse": 0.06632841904647649, "global_step": 920, "grad_norm_max": 1.6363564729690552, "hidden_mse": 0.12368027167394757, "loss": 0.07869644630700351, "lr_blocks": 6.545084971874738e-06, "lr_fusion": 6.066498153718735e-05, "peak_memory_gib": 68.46298551559448}
|
| 270 |
+
{"avg_micro_time_s": 2.105622898535512, "epoch": 14, "flow_mse": 0.0650685330387205, "global_step": 930, "grad_norm_max": 0.9703700542449951, "hidden_mse": 0.12554309023544194, "loss": 0.07762284209020437, "lr_blocks": 6.461858523613684e-06, "lr_fusion": 5.985585137257401e-05, "peak_memory_gib": 68.46298551559448}
|
| 271 |
+
{"avg_micro_time_s": 1.9442461250553607, "epoch": 14, "flow_mse": 0.06523986770771444, "global_step": 940, "grad_norm_max": 1.2431607246398926, "hidden_mse": 0.12500393418595196, "loss": 0.07774026063270867, "lr_blocks": 6.378186779084996e-06, "lr_fusion": 5.90440267166055e-05, "peak_memory_gib": 68.46298551559448}
|
| 272 |
+
{"avg_micro_time_s": 1.828690172272036, "epoch": 14, "flow_mse": 0.05963015616871417, "global_step": 950, "grad_norm_max": 1.5364301204681396, "hidden_mse": 0.11931315260007978, "loss": 0.07156147151254118, "lr_blocks": 6.294095225512604e-06, "lr_fusion": 5.8229729514036705e-05, "peak_memory_gib": 68.46298551559448}
|
| 273 |
+
{"avg_micro_time_s": 1.8559695239790017, "epoch": 14, "flow_mse": 0.05979857188649476, "global_step": 960, "grad_norm_max": 1.4135713577270508, "hidden_mse": 0.11980592980980874, "loss": 0.07177916495129466, "lr_blocks": 6.209609477998339e-06, "lr_fusion": 5.74131823855921e-05, "peak_memory_gib": 68.46298551559448}
|
| 274 |
+
{"avg_micro_time_s": 1.7981327558882185, "epoch": 14, "flow_mse": 0.06091945511288941, "global_step": 970, "grad_norm_max": 1.1532989740371704, "hidden_mse": 0.1190660865046084, "loss": 0.0728260634932667, "lr_blocks": 6.124755271719326e-06, "lr_fusion": 5.6594608567103456e-05, "peak_memory_gib": 68.46298551559448}
|
| 275 |
+
{"avg_micro_time_s": 1.8538059903803514, "epoch": 14, "flow_mse": 0.05959429889917374, "global_step": 980, "grad_norm_max": 1.2059638500213623, "hidden_mse": 0.11662983177229763, "loss": 0.07125728223472834, "lr_blocks": 6.039558454088796e-06, "lr_fusion": 5.577423184847932e-05, "peak_memory_gib": 68.46298551559448}
|
| 276 |
+
{"avg_micro_time_s": 1.7332944313558984, "epoch": 14, "flow_mse": 0.06722243609838188, "global_step": 990, "grad_norm_max": 1.3793468475341797, "hidden_mse": 0.12186120152473451, "loss": 0.07940855622291565, "lr_blocks": 5.954044976882725e-06, "lr_fusion": 5.495227651252315e-05, "peak_memory_gib": 68.46298551559448}
|
| 277 |
+
{"avg_micro_time_s": 1.7073448181181448, "epoch": 15, "flow_mse": 0.05920077268965543, "global_step": 1000, "grad_norm_max": 1.3150396347045898, "hidden_mse": 0.11587047157809138, "loss": 0.07078781956806779, "lr_blocks": 5.8682408883346535e-06, "lr_fusion": 5.4128967273616625e-05, "peak_memory_gib": 68.46298551559448}
|
| 278 |
+
{"avg_micro_time_s": 1.8634853277268122, "epoch": 15, "flow_mse": 0.06008068108931184, "global_step": 1010, "grad_norm_max": 1.0294694900512695, "hidden_mse": 0.11746087549254298, "loss": 0.07182676899246872, "lr_blocks": 5.782172325201155e-06, "lr_fusion": 5.330452921628497e-05, "peak_memory_gib": 68.46298551559448}
|
| 279 |
+
{"avg_micro_time_s": 1.8640984378071153, "epoch": 15, "flow_mse": 0.06041769445873797, "global_step": 1020, "grad_norm_max": 1.2436527013778687, "hidden_mse": 0.11662168577313424, "loss": 0.0720798633992672, "lr_blocks": 5.695865504800328e-06, "lr_fusion": 5.247918773366112e-05, "peak_memory_gib": 68.46298551559448}
|
| 280 |
+
{"avg_micro_time_s": 2.044428478881309, "epoch": 15, "flow_mse": 0.059971056133508685, "global_step": 1030, "grad_norm_max": 0.8764917850494385, "hidden_mse": 0.11762538189068437, "loss": 0.07173359449952842, "lr_blocks": 5.609346717025738e-06, "lr_fusion": 5.165316846586541e-05, "peak_memory_gib": 68.46298551559448}
|
| 281 |
+
{"avg_micro_time_s": 1.7723727050048184, "epoch": 15, "flow_mse": 0.058173052780330185, "global_step": 1040, "grad_norm_max": 1.1820777654647827, "hidden_mse": 0.11490522287786008, "loss": 0.06966357519850135, "lr_blocks": 5.522642316338268e-06, "lr_fusion": 5.0826697238317935e-05, "peak_memory_gib": 68.46298551559448}
|
| 282 |
+
{"avg_micro_time_s": 1.9333946266793647, "epoch": 15, "flow_mse": 0.06083732731640339, "global_step": 1050, "grad_norm_max": 0.8588114380836487, "hidden_mse": 0.11753584127873183, "loss": 0.07259091124869883, "lr_blocks": 5.435778713738292e-06, "lr_fusion": 5e-05, "peak_memory_gib": 68.46298551559448}
|
| 283 |
+
{"avg_micro_time_s": 1.9301579607854364, "epoch": 16, "flow_mse": 0.05984115526080132, "global_step": 1060, "grad_norm_max": 0.7564420104026794, "hidden_mse": 0.116309258248657, "loss": 0.07147208084352315, "lr_blocks": 5.348782368720627e-06, "lr_fusion": 4.917330276168208e-05, "peak_memory_gib": 68.46298551559448}
|
| 284 |
+
{"avg_micro_time_s": 1.6499796213785887, "epoch": 16, "flow_mse": 0.05933821597136557, "global_step": 1070, "grad_norm_max": 1.2188807725906372, "hidden_mse": 0.11562003139406443, "loss": 0.07090021898038686, "lr_blocks": 5.2616797812147205e-06, "lr_fusion": 4.834683153413459e-05, "peak_memory_gib": 68.46298551559448}
|
| 285 |
+
{"avg_micro_time_s": 1.7709604913383374, "epoch": 16, "flow_mse": 0.05792279005981982, "global_step": 1080, "grad_norm_max": 0.9248772263526917, "hidden_mse": 0.11402276260778309, "loss": 0.0693250666372478, "lr_blocks": 5.174497483512506e-06, "lr_fusion": 4.7520812266338885e-05, "peak_memory_gib": 68.46298551559448}
|
| 286 |
+
{"avg_micro_time_s": 1.8961538497242145, "epoch": 16, "flow_mse": 0.05921817822381854, "global_step": 1090, "grad_norm_max": 0.9744265675544739, "hidden_mse": 0.11493033999577165, "loss": 0.07071121269837022, "lr_blocks": 5.087262032186418e-06, "lr_fusion": 4.669547078371504e-05, "peak_memory_gib": 68.46298551559448}
|
| 287 |
+
{"avg_micro_time_s": 1.8375497608751177, "epoch": 16, "flow_mse": 0.05829775258898735, "global_step": 1100, "grad_norm_max": 0.9396805763244629, "hidden_mse": 0.11368074417114259, "loss": 0.06966582736931741, "lr_blocks": 5e-06, "lr_fusion": 4.5871032726383386e-05, "peak_memory_gib": 68.46298551559448}
|
| 288 |
+
{"avg_micro_time_s": 1.8890359286408058, "epoch": 16, "flow_mse": 0.056764368247240786, "global_step": 1110, "grad_norm_max": 1.4962784051895142, "hidden_mse": 0.11246938547119499, "loss": 0.06801130706444383, "lr_blocks": 4.9127379678135825e-06, "lr_fusion": 4.504772348747687e-05, "peak_memory_gib": 68.46298551559448}
|
| 289 |
+
{"avg_micro_time_s": 1.9199464306249867, "epoch": 16, "flow_mse": 0.05812495080754161, "global_step": 1120, "grad_norm_max": 1.0150479078292847, "hidden_mse": 0.11339842304587365, "loss": 0.06946479328908027, "lr_blocks": 4.825502516487497e-06, "lr_fusion": 4.4225768151520694e-05, "peak_memory_gib": 68.46298551559448}
|
| 290 |
+
{"avg_micro_time_s": 1.9528670382205746, "epoch": 17, "flow_mse": 0.0582504840567708, "global_step": 1130, "grad_norm_max": 0.5666173100471497, "hidden_mse": 0.11305413423106075, "loss": 0.0695558977779001, "lr_blocks": 4.738320218785281e-06, "lr_fusion": 4.3405391432896555e-05, "peak_memory_gib": 68.46298551559448}
|
| 291 |
+
{"avg_micro_time_s": 1.8691815458922065, "epoch": 17, "flow_mse": 0.056232606945559385, "global_step": 1140, "grad_norm_max": 1.0409326553344727, "hidden_mse": 0.11095626167953015, "loss": 0.06732823327183723, "lr_blocks": 4.651217631279374e-06, "lr_fusion": 4.2586817614407895e-05, "peak_memory_gib": 68.46298551559448}
|
| 292 |
+
{"avg_micro_time_s": 1.9381392417169991, "epoch": 17, "flow_mse": 0.05920435013249517, "global_step": 1150, "grad_norm_max": 0.8891173005104065, "hidden_mse": 0.11471081618219614, "loss": 0.07067543230950833, "lr_blocks": 4.564221286261709e-06, "lr_fusion": 4.17702704859633e-05, "peak_memory_gib": 68.46298551559448}
|
| 293 |
+
{"avg_micro_time_s": 1.7007469822870918, "epoch": 17, "flow_mse": 0.05892735007219017, "global_step": 1160, "grad_norm_max": 0.9542156457901001, "hidden_mse": 0.11405815165489913, "loss": 0.07033316562883556, "lr_blocks": 4.477357683661734e-06, "lr_fusion": 4.095597328339452e-05, "peak_memory_gib": 68.46298551559448}
|
| 294 |
+
{"avg_micro_time_s": 1.813334083685186, "epoch": 17, "flow_mse": 0.05461530075408519, "global_step": 1170, "grad_norm_max": 0.8323894143104553, "hidden_mse": 0.10710006449371577, "loss": 0.0653253072872758, "lr_blocks": 4.390653282974264e-06, "lr_fusion": 4.0144148627425993e-05, "peak_memory_gib": 68.46298551559448}
|
| 295 |
+
{"avg_micro_time_s": 1.7458943290723257, "epoch": 17, "flow_mse": 0.058256217977032065, "global_step": 1180, "grad_norm_max": 1.1402466297149658, "hidden_mse": 0.11414804682135582, "loss": 0.06967102284543217, "lr_blocks": 4.304134495199675e-06, "lr_fusion": 3.933501846281267e-05, "peak_memory_gib": 68.46298551559448}
|
| 296 |
+
{"avg_micro_time_s": 1.915744307746354, "epoch": 18, "flow_mse": 0.055332719627767805, "global_step": 1190, "grad_norm_max": 0.8607915639877319, "hidden_mse": 0.10889494800940157, "loss": 0.06622221460565925, "lr_blocks": 4.217827674798845e-06, "lr_fusion": 3.852880399766243e-05, "peak_memory_gib": 68.46298551559448}
|
| 297 |
+
{"avg_micro_time_s": 1.9392635020019953, "epoch": 18, "flow_mse": 0.05797289633192122, "global_step": 1200, "grad_norm_max": 1.8679325580596924, "hidden_mse": 0.11137517839670182, "loss": 0.06911041443236172, "lr_blocks": 4.131759111665349e-06, "lr_fusion": 3.772572564296005e-05, "peak_memory_gib": 68.46298551559448}
|
| 298 |
+
{"avg_micro_time_s": 1.7942509921369494, "epoch": 18, "flow_mse": 0.056557841086760166, "global_step": 1210, "grad_norm_max": 0.794858992099762, "hidden_mse": 0.11070305658504367, "loss": 0.06762814656831324, "lr_blocks": 4.045955023117276e-06, "lr_fusion": 3.6926002952309016e-05, "peak_memory_gib": 68.46298551559448}
|
| 299 |
+
{"avg_micro_time_s": 1.6715436912636505, "epoch": 18, "flow_mse": 0.0571835795417428, "global_step": 1220, "grad_norm_max": 0.9790372848510742, "hidden_mse": 0.11226286897435785, "loss": 0.06840986674651504, "lr_blocks": 3.960441545911205e-06, "lr_fusion": 3.612985456190778e-05, "peak_memory_gib": 68.46298551559448}
|
| 300 |
+
{"avg_micro_time_s": 2.107250372883573, "epoch": 18, "flow_mse": 0.0564295522402972, "global_step": 1230, "grad_norm_max": 0.7570212483406067, "hidden_mse": 0.10993242673575879, "loss": 0.06742279478348792, "lr_blocks": 3.875244728280676e-06, "lr_fusion": 3.533749813077677e-05, "peak_memory_gib": 68.46298551559448}
|
| 301 |
+
{"avg_micro_time_s": 2.047389568622748, "epoch": 18, "flow_mse": 0.05746471900492907, "global_step": 1240, "grad_norm_max": 0.8607873320579529, "hidden_mse": 0.11112378425896169, "loss": 0.06857709758915008, "lr_blocks": 3.790390522001662e-06, "lr_fusion": 3.4549150281252636e-05, "peak_memory_gib": 68.46298551559448}
|
| 302 |
+
{"avg_micro_time_s": 1.5449788220197662, "epoch": 18, "flow_mse": 0.05702267340384424, "global_step": 1250, "grad_norm_max": 0.6349190473556519, "hidden_mse": 0.11191629096865655, "loss": 0.06821430274285377, "lr_blocks": 3.705904774487396e-06, "lr_fusion": 3.3765026539765834e-05, "peak_memory_gib": 68.46298551559448}
|
| 303 |
+
{"avg_micro_time_s": 1.8875254256010523, "epoch": 19, "flow_mse": 0.05772090326063335, "global_step": 1260, "grad_norm_max": 0.5843414664268494, "hidden_mse": 0.11196697549894452, "loss": 0.06891760104335845, "lr_blocks": 3.6218132209150047e-06, "lr_fusion": 3.298534127791785e-05, "peak_memory_gib": 68.46298551559448}
|
| 304 |
+
{"avg_micro_time_s": 1.8776245697605192, "epoch": 19, "flow_mse": 0.05313115566968918, "global_step": 1270, "grad_norm_max": 0.44912803173065186, "hidden_mse": 0.1067539899609983, "loss": 0.06380655500106514, "lr_blocks": 3.538141476386317e-06, "lr_fusion": 3.221030765387417e-05, "peak_memory_gib": 68.46298551559448}
|
| 305 |
+
{"avg_micro_time_s": 1.9916057762879065, "epoch": 19, "flow_mse": 0.05610430384986103, "global_step": 1280, "grad_norm_max": 0.7865238785743713, "hidden_mse": 0.10909662246704102, "loss": 0.06701396652497352, "lr_blocks": 3.4549150281252635e-06, "lr_fusion": 3.144013755408895e-05, "peak_memory_gib": 68.46298551559448}
|
| 306 |
+
{"avg_micro_time_s": 1.819823783946049, "epoch": 19, "flow_mse": 0.05514522660523653, "global_step": 1290, "grad_norm_max": 0.9301948547363281, "hidden_mse": 0.10944940159097315, "loss": 0.06609016689471901, "lr_blocks": 3.372159227714218e-06, "lr_fusion": 3.0675041535377405e-05, "peak_memory_gib": 68.46298551559448}
|
| 307 |
+
{"avg_micro_time_s": 2.057221127783123, "epoch": 19, "flow_mse": 0.056626258650794625, "global_step": 1300, "grad_norm_max": 0.5852092504501343, "hidden_mse": 0.1101443249732256, "loss": 0.06764069120399654, "lr_blocks": 3.289899283371657e-06, "lr_fusion": 2.991522876735154e-05, "peak_memory_gib": 68.46298551559448}
|
| 308 |
+
{"avg_micro_time_s": 1.6946213100774914, "epoch": 19, "flow_mse": 0.05420284080319107, "global_step": 1310, "grad_norm_max": 0.49697157740592957, "hidden_mse": 0.10914511308073999, "loss": 0.06511735212989152, "lr_blocks": 3.2081602522734987e-06, "lr_fusion": 2.916090697523549e-05, "peak_memory_gib": 68.46298551559448}
|
| 309 |
+
{"avg_micro_time_s": 1.6702450912911446, "epoch": 19, "flow_mse": 0.05620301379822195, "global_step": 1320, "grad_norm_max": 0.8781030178070068, "hidden_mse": 0.11186566464602948, "loss": 0.067389580514282, "lr_blocks": 3.12696703292044e-06, "lr_fusion": 2.8412282383075363e-05, "peak_memory_gib": 68.46298551559448}
|
| 310 |
+
{"avg_micro_time_s": 1.8671580449008616, "epoch": 20, "flow_mse": 0.057774351164698604, "global_step": 1330, "grad_norm_max": 0.6999236941337585, "hidden_mse": 0.1124896040186286, "loss": 0.06902331141754985, "lr_blocks": 3.0463443575536324e-06, "lr_fusion": 2.766955965735968e-05, "peak_memory_gib": 68.46298551559448}
|
| 311 |
+
{"avg_micro_time_s": 1.7934378478326836, "epoch": 20, "flow_mse": 0.05308823841623962, "global_step": 1340, "grad_norm_max": 0.6592912673950195, "hidden_mse": 0.10645070374011995, "loss": 0.06373330899514258, "lr_blocks": 2.966316784621e-06, "lr_fusion": 2.693294185106562e-05, "peak_memory_gib": 68.46298551559448}
|
| 312 |
+
{"avg_micro_time_s": 1.983334714593366, "epoch": 20, "flow_mse": 0.05545833441428841, "global_step": 1350, "grad_norm_max": 0.619492769241333, "hidden_mse": 0.10996899055317044, "loss": 0.06645523337647319, "lr_blocks": 2.886908691296504e-06, "lr_fusion": 2.6202630348146324e-05, "peak_memory_gib": 68.46298551559448}
|
| 313 |
+
{"avg_micro_time_s": 1.707897415009211, "epoch": 20, "flow_mse": 0.05316436188295484, "global_step": 1360, "grad_norm_max": 0.7670770883560181, "hidden_mse": 0.10698874769732357, "loss": 0.06386323706246913, "lr_blocks": 2.8081442660546126e-06, "lr_fusion": 2.547882480847461e-05, "peak_memory_gib": 68.46298551559448}
|
| 314 |
+
{"avg_micro_time_s": 1.8228236074690358, "epoch": 20, "flow_mse": 0.05519023141823709, "global_step": 1370, "grad_norm_max": 0.5514799356460571, "hidden_mse": 0.10873732827603817, "loss": 0.06606396441347898, "lr_blocks": 2.7300475013022666e-06, "lr_fusion": 2.476172311325783e-05, "peak_memory_gib": 68.46298551559448}
|
| 315 |
+
{"avg_micro_time_s": 1.8020589397929143, "epoch": 20, "flow_mse": 0.05520334360189736, "global_step": 1380, "grad_norm_max": 0.7075578570365906, "hidden_mse": 0.10943609643727542, "loss": 0.0661469531711191, "lr_blocks": 2.6526421860705474e-06, "lr_fusion": 2.405152131093926e-05, "peak_memory_gib": 68.46298551559448}
|
| 316 |
+
{"avg_micro_time_s": 2.1456252824849797, "epoch": 21, "flow_mse": 0.055343919666484, "global_step": 1390, "grad_norm_max": 0.4367266297340393, "hidden_mse": 0.10879806661978364, "loss": 0.0662237268872559, "lr_blocks": 2.5759518987683154e-06, "lr_fusion": 2.3348413563600325e-05, "peak_memory_gib": 68.46298551559448}
|
| 317 |
+
{"avg_micro_time_s": 1.9225107472884702, "epoch": 21, "flow_mse": 0.05379870082251728, "global_step": 1400, "grad_norm_max": 0.45527708530426025, "hidden_mse": 0.10747067201882601, "loss": 0.0645457681734115, "lr_blocks": 2.5000000000000015e-06, "lr_fusion": 2.2652592093878666e-05, "peak_memory_gib": 68.46298551559448}
|
| 318 |
+
{"avg_micro_time_s": 1.8355080222332618, "epoch": 21, "flow_mse": 0.05405601593665779, "global_step": 1410, "grad_norm_max": 0.578701376914978, "hidden_mse": 0.10813164105638862, "loss": 0.06486918060109019, "lr_blocks": 2.424809625449729e-06, "lr_fusion": 2.196424713241637e-05, "peak_memory_gib": 68.46298551559448}
|
| 319 |
+
{"avg_micro_time_s": 1.8781015649496113, "epoch": 21, "flow_mse": 0.05536241387017071, "global_step": 1420, "grad_norm_max": 0.38821011781692505, "hidden_mse": 0.11099598519504071, "loss": 0.06646201252005994, "lr_blocks": 2.3504036788339763e-06, "lr_fusion": 2.128356686585282e-05, "peak_memory_gib": 68.46298551559448}
|
| 320 |
+
{"avg_micro_time_s": 1.7020483079002589, "epoch": 21, "flow_mse": 0.05298192100599408, "global_step": 1430, "grad_norm_max": 0.5125593543052673, "hidden_mse": 0.10700325099751354, "loss": 0.06368224625475705, "lr_blocks": 2.2768048249248648e-06, "lr_fusion": 2.061073738537635e-05, "peak_memory_gib": 68.46298551559448}
|
| 321 |
+
{"avg_micro_time_s": 1.8793439952991322, "epoch": 21, "flow_mse": 0.05324047850444913, "global_step": 1440, "grad_norm_max": 0.36654290556907654, "hidden_mse": 0.10668059764429927, "loss": 0.06390853836201131, "lr_blocks": 2.204035482646267e-06, "lr_fusion": 1.9945942635848748e-05, "peak_memory_gib": 68.46298551559448}
|
| 322 |
+
{"avg_micro_time_s": 1.697708303979016, "epoch": 21, "flow_mse": 0.055548487138003116, "global_step": 1450, "grad_norm_max": 0.5792702436447144, "hidden_mse": 0.11123776510357858, "loss": 0.06667226417921483, "lr_blocks": 2.132117818244771e-06, "lr_fusion": 1.928936436551661e-05, "peak_memory_gib": 68.46298551559448}
|
| 323 |
+
{"avg_micro_time_s": 1.8742496912454953, "epoch": 22, "flow_mse": 0.05278900931589306, "global_step": 1460, "grad_norm_max": 0.41618072986602783, "hidden_mse": 0.10622980045154691, "loss": 0.06341198971495032, "lr_blocks": 2.061073738537635e-06, "lr_fusion": 1.8641182076323148e-05, "peak_memory_gib": 68.46298551559448}
|
| 324 |
+
{"avg_micro_time_s": 2.0268068581106493, "epoch": 22, "flow_mse": 0.05427227606996894, "global_step": 1470, "grad_norm_max": 0.38854727149009705, "hidden_mse": 0.10826945109292865, "loss": 0.06509922100231051, "lr_blocks": 1.990924884239758e-06, "lr_fusion": 1.800157297483417e-05, "peak_memory_gib": 68.46298551559448}
|
| 325 |
+
{"avg_micro_time_s": 1.6323719802472625, "epoch": 22, "flow_mse": 0.05504520414397121, "global_step": 1480, "grad_norm_max": 0.42108166217803955, "hidden_mse": 0.11108364090323448, "loss": 0.06615356872789561, "lr_blocks": 1.9216926233717087e-06, "lr_fusion": 1.7370711923791567e-05, "peak_memory_gib": 68.46298551559448}
|
| 326 |
+
{"avg_micro_time_s": 1.7635257252361045, "epoch": 22, "flow_mse": 0.05449590594507754, "global_step": 1490, "grad_norm_max": 0.2980191111564636, "hidden_mse": 0.10923452377319337, "loss": 0.06541935848072171, "lr_blocks": 1.8533980447508138e-06, "lr_fusion": 1.6748771394307585e-05, "peak_memory_gib": 68.46298551559448}
|
| 327 |
+
{"avg_micro_time_s": 1.7368722179773615, "epoch": 22, "flow_mse": 0.05059092580340803, "global_step": 1500, "grad_norm_max": 0.3388434052467346, "hidden_mse": 0.10435767890885472, "loss": 0.06102669364772737, "lr_blocks": 1.7860619515673034e-06, "lr_fusion": 1.6135921418712956e-05, "peak_memory_gib": 68.46298551559448}
|
| 328 |
+
{"avg_micro_time_s": 2.083392089595145, "epoch": 22, "flow_mse": 0.05451004351489246, "global_step": 1510, "grad_norm_max": 0.31937962770462036, "hidden_mse": 0.10862588789314032, "loss": 0.0653726323042065, "lr_blocks": 1.7197048550474643e-06, "lr_fusion": 1.553232954407171e-05, "peak_memory_gib": 68.46298551559448}
|
| 329 |
+
{"avg_micro_time_s": 1.7850786256254652, "epoch": 23, "flow_mse": 0.05411349283531308, "global_step": 1520, "grad_norm_max": 0.43102946877479553, "hidden_mse": 0.1085999496281147, "loss": 0.06497348803095519, "lr_blocks": 1.6543469682057105e-06, "lr_fusion": 1.4938160786375572e-05, "peak_memory_gib": 68.46298551559448}
|
| 330 |
+
{"avg_micro_time_s": 1.6930420300239348, "epoch": 23, "flow_mse": 0.05289990794844926, "global_step": 1530, "grad_norm_max": 0.3967125415802002, "hidden_mse": 0.10759261148050428, "loss": 0.06365916887298227, "lr_blocks": 1.5900081996875083e-06, "lr_fusion": 1.435357758543015e-05, "peak_memory_gib": 68.46298551559448}
|
| 331 |
+
{"avg_micro_time_s": 1.8721173314916086, "epoch": 23, "flow_mse": 0.05183863551355899, "global_step": 1540, "grad_norm_max": 0.65227872133255, "hidden_mse": 0.10619828049093485, "loss": 0.062458463944494724, "lr_blocks": 1.5267081477050132e-06, "lr_fusion": 1.3778739760445552e-05, "peak_memory_gib": 68.46298551559448}
|
| 332 |
+
{"avg_micro_time_s": 1.9260684134074837, "epoch": 23, "flow_mse": 0.05517242061905563, "global_step": 1550, "grad_norm_max": 0.3173617720603943, "hidden_mse": 0.11037433873862029, "loss": 0.06620985479094088, "lr_blocks": 1.4644660940672628e-06, "lr_fusion": 1.3213804466343421e-05, "peak_memory_gib": 68.46298551559448}
|
| 333 |
+
{"avg_micro_time_s": 2.00732960163441, "epoch": 23, "flow_mse": 0.052165593765676026, "global_step": 1560, "grad_norm_max": 0.48366814851760864, "hidden_mse": 0.10552244568243624, "loss": 0.06271783891133964, "lr_blocks": 1.4033009983067454e-06, "lr_fusion": 1.2658926150792322e-05, "peak_memory_gib": 68.46298551559448}
|
| 334 |
+
{"avg_micro_time_s": 2.0028102155803937, "epoch": 23, "flow_mse": 0.054785998072475195, "global_step": 1570, "grad_norm_max": 0.32774579524993896, "hidden_mse": 0.1094754220917821, "loss": 0.0657335405703634, "lr_blocks": 1.3432314919041478e-06, "lr_fusion": 1.2114256511983274e-05, "peak_memory_gib": 68.46298551559448}
|
| 335 |
+
{"avg_micro_time_s": 1.8898658655904002, "epoch": 23, "flow_mse": 0.05355181619524956, "global_step": 1580, "grad_norm_max": 0.29386377334594727, "hidden_mse": 0.10809278432279826, "loss": 0.06436109468340874, "lr_blocks": 1.2842758726130283e-06, "lr_fusion": 1.157994445715706e-05, "peak_memory_gib": 68.46298551559448}
|
| 336 |
+
{"avg_micro_time_s": 1.8055248979391765, "epoch": 24, "flow_mse": 0.05223282501101494, "global_step": 1590, "grad_norm_max": 0.30623888969421387, "hidden_mse": 0.10633350862190127, "loss": 0.06286617605946958, "lr_blocks": 1.22645209888614e-06, "lr_fusion": 1.1056136061894384e-05, "peak_memory_gib": 68.46298551559448}
|
| 337 |
+
{"avg_micro_time_s": 1.697861864778679, "epoch": 24, "flow_mse": 0.05407982333563269, "global_step": 1600, "grad_norm_max": 0.41349685192108154, "hidden_mse": 0.1090579808689654, "loss": 0.06498562130145728, "lr_blocks": 1.1697777844051105e-06, "lr_fusion": 1.0542974530180327e-05, "peak_memory_gib": 68.46298551559448}
|
| 338 |
+
{"avg_micro_time_s": 1.858836678104126, "epoch": 24, "flow_mse": 0.0515467718243599, "global_step": 1610, "grad_norm_max": 0.2459108531475067, "hidden_mse": 0.10587860029190779, "loss": 0.06213463232852519, "lr_blocks": 1.1142701927151456e-06, "lr_fusion": 1.0040600155253765e-05, "peak_memory_gib": 68.46298551559448}
|
| 339 |
+
{"avg_micro_time_s": 1.7442898534587585, "epoch": 24, "flow_mse": 0.05539359417743981, "global_step": 1620, "grad_norm_max": 0.4176762104034424, "hidden_mse": 0.11120789768174291, "loss": 0.06651438400149345, "lr_blocks": 1.0599462319663906e-06, "lr_fusion": 9.549150281252633e-06, "peak_memory_gib": 68.46298551559448}
|
| 340 |
+
{"avg_micro_time_s": 1.9312700220412808, "epoch": 24, "flow_mse": 0.05194714558310807, "global_step": 1630, "grad_norm_max": 0.27055931091308594, "hidden_mse": 0.10536178648471833, "loss": 0.06248332438990474, "lr_blocks": 1.006822449763537e-06, "lr_fusion": 9.068759265665384e-06, "peak_memory_gib": 68.46298551559448}
|
| 341 |
+
{"avg_micro_time_s": 1.9848020562625608, "epoch": 24, "flow_mse": 0.05159918488934637, "global_step": 1640, "grad_norm_max": 0.2410484105348587, "hidden_mse": 0.10558492876589298, "loss": 0.06215767813846469, "lr_blocks": 9.549150281252633e-07, "lr_fusion": 8.599558442598998e-06, "peak_memory_gib": 68.46298551559448}
|
| 342 |
+
{"avg_micro_time_s": 1.9862558376597008, "epoch": 24, "flow_mse": 0.051943618524819615, "global_step": 1650, "grad_norm_max": 0.19621193408966064, "hidden_mse": 0.10532341394573451, "loss": 0.062475959723815325, "lr_blocks": 9.042397785550405e-07, "lr_fusion": 8.141676086873572e-06, "peak_memory_gib": 68.46298551559448}
|
| 343 |
+
{"avg_micro_time_s": 1.90772508844384, "epoch": 25, "flow_mse": 0.05241604868788272, "global_step": 1660, "grad_norm_max": 0.24477998912334442, "hidden_mse": 0.10615330189466476, "loss": 0.06303137941285968, "lr_blocks": 8.54812137224792e-07, "lr_fusion": 7.695237378953223e-06, "peak_memory_gib": 68.46298551559448}
|
| 344 |
+
{"avg_micro_time_s": 2.0734705874245267, "epoch": 25, "flow_mse": 0.052382499817758804, "global_step": 1670, "grad_norm_max": 0.18695969879627228, "hidden_mse": 0.10536184627562761, "loss": 0.06291868472471833, "lr_blocks": 8.066471602728804e-07, "lr_fusion": 7.260364370723044e-06, "peak_memory_gib": 68.46298551559448}
|
| 345 |
+
{"avg_micro_time_s": 1.784898155926203, "epoch": 25, "flow_mse": 0.05266823512502015, "global_step": 1680, "grad_norm_max": 0.30488482117652893, "hidden_mse": 0.10819421242922544, "loss": 0.0634876563679427, "lr_blocks": 7.597595192178702e-07, "lr_fusion": 6.837175952121306e-06, "peak_memory_gib": 68.46298551559448}
|
| 346 |
+
{"avg_micro_time_s": 1.7039467210503063, "epoch": 25, "flow_mse": 0.053952160244807604, "global_step": 1690, "grad_norm_max": 0.21721898019313812, "hidden_mse": 0.10940732667222619, "loss": 0.06489289305172861, "lr_blocks": 7.141634964894389e-07, "lr_fusion": 6.425787818636131e-06, "peak_memory_gib": 68.46298551559448}
|
| 347 |
+
{"avg_micro_time_s": 1.7925323831252173, "epoch": 25, "flow_mse": 0.054602593369781974, "global_step": 1700, "grad_norm_max": 0.2606789171695709, "hidden_mse": 0.11060061100870372, "loss": 0.06566265453584492, "lr_blocks": 6.698729810778065e-07, "lr_fusion": 6.026312439675552e-06, "peak_memory_gib": 68.46298551559448}
|
| 348 |
+
{"avg_micro_time_s": 1.8506990019581282, "epoch": 25, "flow_mse": 0.05084327077493072, "global_step": 1710, "grad_norm_max": 0.18723498284816742, "hidden_mse": 0.10409033391624689, "loss": 0.06125230453908444, "lr_blocks": 6.269014643030214e-07, "lr_fusion": 5.6388590278194096e-06, "peak_memory_gib": 68.46298551559448}
|
| 349 |
+
{"avg_micro_time_s": 1.696594527331763, "epoch": 26, "flow_mse": 0.04985265554860235, "global_step": 1720, "grad_norm_max": 0.16147896647453308, "hidden_mse": 0.10344723165035248, "loss": 0.060197379020974044, "lr_blocks": 5.852620357053651e-07, "lr_fusion": 5.263533508961827e-06, "peak_memory_gib": 68.46298551559448}
|
| 350 |
+
{"avg_micro_time_s": 1.7064527680224275, "epoch": 26, "flow_mse": 0.052878752723336225, "global_step": 1730, "grad_norm_max": 0.21210241317749023, "hidden_mse": 0.10743851037696005, "loss": 0.0636226034257561, "lr_blocks": 5.449673790581611e-07, "lr_fusion": 4.900438493352055e-06, "peak_memory_gib": 68.46298551559448}
|
| 351 |
+
{"avg_micro_time_s": 1.9300253918670933, "epoch": 26, "flow_mse": 0.05500989542342723, "global_step": 1740, "grad_norm_max": 0.18679021298885345, "hidden_mse": 0.11223258469253779, "loss": 0.06623315447941423, "lr_blocks": 5.06029768504166e-07, "lr_fusion": 4.549673247541875e-06, "peak_memory_gib": 68.46298551559448}
|
| 352 |
+
{"avg_micro_time_s": 1.6863267055538018, "epoch": 26, "flow_mse": 0.05216321973130107, "global_step": 1750, "grad_norm_max": 0.1739465296268463, "hidden_mse": 0.10685156406834723, "loss": 0.06284837638959288, "lr_blocks": 4.6846106481675035e-07, "lr_fusion": 4.2113336672471245e-06, "peak_memory_gib": 68.46298551559448}
|
| 353 |
+
{"avg_micro_time_s": 1.9015966636099622, "epoch": 26, "flow_mse": 0.05238161133602262, "global_step": 1760, "grad_norm_max": 0.16062317788600922, "hidden_mse": 0.1064825899899006, "loss": 0.06302987090311944, "lr_blocks": 4.322727117869951e-07, "lr_fusion": 3.885512251130763e-06, "peak_memory_gib": 68.46298551559448}
|
| 354 |
+
{"avg_micro_time_s": 1.89506446889136, "epoch": 26, "flow_mse": 0.05142894261516631, "global_step": 1770, "grad_norm_max": 0.19871976971626282, "hidden_mse": 0.10540924789384008, "loss": 0.06196986767463386, "lr_blocks": 3.9747573273779816e-07, "lr_fusion": 3.5722980755146517e-06, "peak_memory_gib": 68.46298551559448}
|
| 355 |
+
{"avg_micro_time_s": 2.140559560073598, "epoch": 26, "flow_mse": 0.052294114232063295, "global_step": 1780, "grad_norm_max": 0.13666212558746338, "hidden_mse": 0.10540376659482718, "loss": 0.06283449097536505, "lr_blocks": 3.6408072716606346e-07, "lr_fusion": 3.271776770026963e-06, "peak_memory_gib": 68.46298551559448}
|
| 356 |
+
{"avg_micro_time_s": 1.9869108784609126, "epoch": 27, "flow_mse": 0.052665405767038466, "global_step": 1790, "grad_norm_max": 0.1539895087480545, "hidden_mse": 0.1076177385635674, "loss": 0.06342717967927457, "lr_blocks": 3.320978675139919e-07, "lr_fusion": 2.9840304941919415e-06, "peak_memory_gib": 68.46298551559448}
|
| 357 |
+
{"avg_micro_time_s": 1.8483484975920874, "epoch": 27, "flow_mse": 0.05281953318044544, "global_step": 1800, "grad_norm_max": 0.1717279851436615, "hidden_mse": 0.10747445961460472, "loss": 0.06356697916053236, "lr_blocks": 3.015368960704584e-07, "lr_fusion": 2.7091379149682685e-06, "peak_memory_gib": 68.46298551559448}
|
| 358 |
+
{"avg_micro_time_s": 1.7725311637754204, "epoch": 27, "flow_mse": 0.053238363936543465, "global_step": 1810, "grad_norm_max": 0.129792258143425, "hidden_mse": 0.10810153055936099, "loss": 0.06404851726256311, "lr_blocks": 2.724071220034158e-07, "lr_fusion": 2.4471741852423237e-06, "peak_memory_gib": 68.46298551559448}
|
| 359 |
+
{"avg_micro_time_s": 1.9046508581523085, "epoch": 27, "flow_mse": 0.05027513452805579, "global_step": 1820, "grad_norm_max": 0.13140571117401123, "hidden_mse": 0.1037200196646154, "loss": 0.06064713676460087, "lr_blocks": 2.447174185242324e-07, "lr_fusion": 2.1982109232821178e-06, "peak_memory_gib": 68.46298551559448}
|
| 360 |
+
{"avg_micro_time_s": 1.7696023706230335, "epoch": 27, "flow_mse": 0.053843728452920916, "global_step": 1830, "grad_norm_max": 0.10490740090608597, "hidden_mse": 0.10948641048744322, "loss": 0.06479236944578588, "lr_blocks": 2.1847622018482283e-07, "lr_fusion": 1.962316193157593e-06, "peak_memory_gib": 68.46298551559448}
|
| 361 |
+
{"avg_micro_time_s": 1.8017618429686992, "epoch": 27, "flow_mse": 0.05273311133496464, "global_step": 1840, "grad_norm_max": 0.1325991004705429, "hidden_mse": 0.10784135181456805, "loss": 0.06351724686101079, "lr_blocks": 1.9369152030840553e-07, "lr_fusion": 1.7395544861325718e-06, "peak_memory_gib": 68.46298551559448}
|
| 362 |
+
{"avg_micro_time_s": 1.8663546495808987, "epoch": 28, "flow_mse": 0.05010635161306709, "global_step": 1850, "grad_norm_max": 0.13209085166454315, "hidden_mse": 0.10319266710430385, "loss": 0.06042561884969473, "lr_blocks": 1.7037086855465902e-07, "lr_fusion": 1.5299867030334814e-06, "peak_memory_gib": 68.46298551559448}
|
| 363 |
+
{"avg_micro_time_s": 2.023102572160133, "epoch": 28, "flow_mse": 0.0542366745416075, "global_step": 1860, "grad_norm_max": 0.09557513147592545, "hidden_mse": 0.10798764936625958, "loss": 0.06503543937578797, "lr_blocks": 1.4852136862001766e-07, "lr_fusion": 1.333670137599713e-06, "peak_memory_gib": 68.46298551559448}
|
| 364 |
+
{"avg_micro_time_s": 1.9713299808965532, "epoch": 28, "flow_mse": 0.050487085804343225, "global_step": 1870, "grad_norm_max": 0.09084595739841461, "hidden_mse": 0.1046027010306716, "loss": 0.06094735627993941, "lr_blocks": 1.2814967607382433e-07, "lr_fusion": 1.1506584608200367e-06, "peak_memory_gib": 68.46298551559448}
|
| 365 |
+
{"avg_micro_time_s": 1.765306223012158, "epoch": 28, "flow_mse": 0.052992650866508485, "global_step": 1880, "grad_norm_max": 0.1068035289645195, "hidden_mse": 0.10854229601100088, "loss": 0.06384688057005405, "lr_blocks": 1.0926199633097156e-07, "lr_fusion": 9.810017062595322e-07, "peak_memory_gib": 68.46298551559448}
|
| 366 |
+
{"avg_micro_time_s": 1.8488479020117667, "epoch": 28, "flow_mse": 0.05185321508906782, "global_step": 1890, "grad_norm_max": 0.0908748134970665, "hidden_mse": 0.10693074595183134, "loss": 0.06254628994502127, "lr_blocks": 9.186408276168012e-08, "lr_fusion": 8.247462563808817e-07, "peak_memory_gib": 68.46298551559448}
|
| 367 |
+
{"avg_micro_time_s": 1.835118718787271, "epoch": 28, "flow_mse": 0.050304826558567585, "global_step": 1900, "grad_norm_max": 0.07472742348909378, "hidden_mse": 0.10459219636395574, "loss": 0.06076404661871493, "lr_blocks": 7.59612349389599e-08, "lr_fusion": 6.819348298638839e-07, "peak_memory_gib": 68.46298551559448}
|
| 368 |
+
{"avg_micro_time_s": 1.8683127601994785, "epoch": 28, "flow_mse": 0.05247240727767349, "global_step": 1910, "grad_norm_max": 0.0885327011346817, "hidden_mse": 0.1072016378864646, "loss": 0.06319257123395801, "lr_blocks": 6.15582970243117e-08, "lr_fusion": 5.526064699265753e-07, "peak_memory_gib": 68.46298551559448}
|
| 369 |
+
{"avg_micro_time_s": 1.6088743058397086, "epoch": 29, "flow_mse": 0.05225406056270004, "global_step": 1920, "grad_norm_max": 0.06508878618478775, "hidden_mse": 0.10751305781304837, "loss": 0.06300536664202809, "lr_blocks": 4.865965629214819e-08, "lr_fusion": 4.367965336512403e-07, "peak_memory_gib": 68.46298551559448}
|
| 370 |
+
{"avg_micro_time_s": 1.6788693653565134, "epoch": 29, "flow_mse": 0.05310070822015405, "global_step": 1930, "grad_norm_max": 0.09013048559427261, "hidden_mse": 0.10843602363020183, "loss": 0.0639443107880652, "lr_blocks": 3.726924179339009e-08, "lr_fusion": 3.3453668231809286e-07, "peak_memory_gib": 68.46298551559448}
|
| 371 |
+
{"avg_micro_time_s": 1.8483045230590507, "epoch": 29, "flow_mse": 0.05225380118936301, "global_step": 1940, "grad_norm_max": 0.05455300584435463, "hidden_mse": 0.10687568383291364, "loss": 0.06294136978685856, "lr_blocks": 2.7390523158633552e-08, "lr_fusion": 2.458548727494292e-07, "peak_memory_gib": 68.46298551559448}
|
| 372 |
+
{"avg_micro_time_s": 2.07668071983353, "epoch": 29, "flow_mse": 0.052197744604200125, "global_step": 1950, "grad_norm_max": 0.06954554468393326, "hidden_mse": 0.10710220159962774, "loss": 0.06290796506218613, "lr_blocks": 1.9026509541272276e-08, "lr_fusion": 1.7077534966650766e-07, "peak_memory_gib": 68.46298551559448}
|
| 373 |
+
{"avg_micro_time_s": 1.890728423825931, "epoch": 29, "flow_mse": 0.052322149462997916, "global_step": 1960, "grad_norm_max": 0.06443354487419128, "hidden_mse": 0.10821447782218457, "loss": 0.06314359758980573, "lr_blocks": 1.2179748700879013e-08, "lr_fusion": 1.0931863906127327e-07, "peak_memory_gib": 68.46298551559448}
|
| 374 |
+
{"avg_micro_time_s": 1.7953088903203025, "epoch": 29, "flow_mse": 0.05093962624669075, "global_step": 1970, "grad_norm_max": 0.07561659812927246, "hidden_mse": 0.10430180076509715, "loss": 0.06136980666778982, "lr_blocks": 6.852326227130835e-09, "lr_fusion": 6.150154258476315e-08, "peak_memory_gib": 68.46298551559448}
|
| 375 |
+
{"avg_micro_time_s": 1.842160684411647, "epoch": 29, "flow_mse": 0.053311998723074794, "global_step": 1980, "grad_norm_max": 0.04818754270672798, "hidden_mse": 0.10754476888105274, "loss": 0.06406647614203394, "lr_blocks": 3.0458649045211897e-09, "lr_fusion": 2.7337132953697554e-08, "peak_memory_gib": 68.46298551559448}
|
| 376 |
+
{"avg_micro_time_s": 1.7273795908869944, "epoch": 30, "flow_mse": 0.051652479777112605, "global_step": 1990, "grad_norm_max": 0.061520446091890335, "hidden_mse": 0.10536492271348835, "loss": 0.06218897206708789, "lr_blocks": 7.615242180436521e-10, "lr_fusion": 6.834750376549792e-09, "peak_memory_gib": 68.46298551559448}
|
| 377 |
+
{"avg_micro_time_s": 1.9548408255825054, "epoch": 30, "flow_mse": 0.05280004227533937, "global_step": 2000, "grad_norm_max": 0.054393548518419266, "hidden_mse": 0.10706793488934636, "loss": 0.06350683607161045, "lr_blocks": 0.0, "lr_fusion": 0.0, "peak_memory_gib": 68.46298551559448}
|
| 378 |
+
swanlab: 📁 View project at
|
| 379 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 380 |
+
swanlab: 🚀 View run pre...8-prompts1000-seed0-bs32x8-from-scratch at
|
| 381 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/4kxa9v7b
|
| 382 |
+
swanlab: Upload complete: 72320 records
|
| 383 |
+
[2026-07-25T18:18:11Z] 训练完成，开始同步 NVMe 数据和 checkpoint。
|
| 384 |
+
同步数据：/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0 -> /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/data/self_forcing_predictor_v4_1000_seed0
|
| 385 |
+
|
| 386 |
+
Number of files: 41,040 (reg: 36,030, dir: 5,010)
|
| 387 |
+
Number of created files: 41,039 (reg: 36,030, dir: 5,009)
|
| 388 |
+
Number of deleted files: 0
|
| 389 |
+
Number of regular files transferred: 36,030
|
| 390 |
+
Total file size: 851.31G bytes
|
| 391 |
+
Total transferred file size: 851.31G bytes
|
| 392 |
+
Literal data: 851.31G bytes
|
| 393 |
+
Matched data: 0 bytes
|
| 394 |
+
File list size: 524.25K
|
| 395 |
+
File list generation time: 0.001 seconds
|
| 396 |
+
File list transfer time: 0.000 seconds
|
| 397 |
+
Total bytes sent: 851.52G
|
| 398 |
+
Total bytes received: 727.73K
|
| 399 |
+
|
| 400 |
+
sent 851.52G bytes received 727.73K bytes 152.34M bytes/sec
|
| 401 |
+
total size is 851.31G speedup is 1.00
|
| 402 |
+
同步训练产物：/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0 -> /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/predictor_v4_1000_seed0
|
| 403 |
+
|
| 404 |
+
Number of files: 44 (reg: 38, dir: 6)
|
| 405 |
+
Number of created files: 43 (reg: 38, dir: 5)
|
| 406 |
+
Number of deleted files: 0
|
| 407 |
+
Number of regular files transferred: 38
|
| 408 |
+
Total file size: 5.45G bytes
|
| 409 |
+
Total transferred file size: 5.45G bytes
|
| 410 |
+
Literal data: 5.45G bytes
|
| 411 |
+
Matched data: 0 bytes
|
| 412 |
+
File list size: 0
|
| 413 |
+
File list generation time: 0.001 seconds
|
| 414 |
+
File list transfer time: 0.000 seconds
|
| 415 |
+
Total bytes sent: 5.45G
|
| 416 |
+
Total bytes received: 773
|
| 417 |
+
|
| 418 |
+
sent 5.45G bytes received 773 bytes 436.09M bytes/sec
|
| 419 |
+
total size is 5.45G speedup is 1.00
|
| 420 |
+
NVMe 产物已完整同步回 Self-Forcing 项目。
|
| 421 |
+
[2026-07-25T19:52:55Z] 训练与同步全部完成。
|
checkpoints/predictor_v4_1000_seed0/training_and_sync_before_fresh_bs32_20260725.log
ADDED
|
@@ -0,0 +1,504 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-25T13:16:41Z] 从完整 NVMe 数据启动 8 GPU Predictor-v4 训练。
|
| 2 |
+
WARNING:__main__:
|
| 3 |
+
*****************************************
|
| 4 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 5 |
+
*****************************************
|
| 6 |
+
[rank3]:[W725 13:16:57.092135206 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 7 |
+
[rank0]:[W725 13:16:57.284129232 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 8 |
+
[rank1]:[W725 13:16:57.347025266 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 9 |
+
[rank5]:[W725 13:16:57.459626322 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 10 |
+
[rank7]:[W725 13:16:57.612025855 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 11 |
+
[rank6]:[W725 13:16:57.612414421 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 12 |
+
[rank4]:[W725 13:16:57.633515615 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 13 |
+
[rank2]:[W725 13:16:57.650871076 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 14 |
+
swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use
|
| 15 |
+
`swanlab login --relogin` to force relogin
|
| 16 |
+
swanlab: Tracking run with swanlab version 0.9.0
|
| 17 |
+
swanlab: 💾 Run data saved at
|
| 18 |
+
/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/
|
| 19 |
+
run-20260725_131749-4h9zkrjp
|
| 20 |
+
swanlab: 📁 View project at
|
| 21 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 22 |
+
swanlab: 🚀 View run pre...ure-blocks1-28-prompts1000-seed0-bs16x8 at
|
| 23 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/4h9zkrjp
|
| 24 |
+
{"swanlab_run_id": "4h9zkrjp"}
|
| 25 |
+
{
|
| 26 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 27 |
+
"prompt_count": 1000,
|
| 28 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 29 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 30 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 31 |
+
"max_records": null,
|
| 32 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 33 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 34 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 35 |
+
"strict_teacher_load": true,
|
| 36 |
+
"source_block_ids": [
|
| 37 |
+
1,
|
| 38 |
+
28
|
| 39 |
+
],
|
| 40 |
+
"timestep_shift": 5.0,
|
| 41 |
+
"local_attn_size": -1,
|
| 42 |
+
"sink_size": 0,
|
| 43 |
+
"latent_height": 60,
|
| 44 |
+
"latent_width": 104,
|
| 45 |
+
"max_steps": 2000,
|
| 46 |
+
"batch_size": 16,
|
| 47 |
+
"gradient_accumulation_steps": 1,
|
| 48 |
+
"num_workers": 0,
|
| 49 |
+
"pin_memory": false,
|
| 50 |
+
"persistent_workers": false,
|
| 51 |
+
"fusion_lr": 0.0001,
|
| 52 |
+
"blocks_lr": 1e-05,
|
| 53 |
+
"beta1": 0.9,
|
| 54 |
+
"beta2": 0.95,
|
| 55 |
+
"weight_decay": 0.01,
|
| 56 |
+
"scheduler_warmup_steps": 100,
|
| 57 |
+
"block_warmup_steps": 100,
|
| 58 |
+
"hidden_loss_weight": 0.1,
|
| 59 |
+
"flow_loss_weight": 1.0,
|
| 60 |
+
"grad_clip": 1.0,
|
| 61 |
+
"fp32_trainable_params": true,
|
| 62 |
+
"gradient_checkpointing": false,
|
| 63 |
+
"find_unused_parameters": false,
|
| 64 |
+
"log_every": 10,
|
| 65 |
+
"save_every": 100,
|
| 66 |
+
"keep_snapshots": 20,
|
| 67 |
+
"resume": null,
|
| 68 |
+
"use_wandb": false,
|
| 69 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 70 |
+
"wandb_entity": null,
|
| 71 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 72 |
+
"use_swanlab": true,
|
| 73 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 74 |
+
"swanlab_workspace": null,
|
| 75 |
+
"swanlab_experiment": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs16x8",
|
| 76 |
+
"swanlab_description": "Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training",
|
| 77 |
+
"swanlab_tags": [
|
| 78 |
+
"predictor-v4",
|
| 79 |
+
"self-forcing",
|
| 80 |
+
"wan2.1-1.3b",
|
| 81 |
+
"offline-prefeature",
|
| 82 |
+
"blocks-1-28",
|
| 83 |
+
"8gpu",
|
| 84 |
+
"seed0"
|
| 85 |
+
],
|
| 86 |
+
"swanlab_mode": "cloud",
|
| 87 |
+
"supervision_pairs": [
|
| 88 |
+
[
|
| 89 |
+
0,
|
| 90 |
+
1
|
| 91 |
+
],
|
| 92 |
+
[
|
| 93 |
+
1,
|
| 94 |
+
2
|
| 95 |
+
],
|
| 96 |
+
[
|
| 97 |
+
2,
|
| 98 |
+
3
|
| 99 |
+
]
|
| 100 |
+
],
|
| 101 |
+
"manifest_records": 6000,
|
| 102 |
+
"dataset_pairs": 18000,
|
| 103 |
+
"steps_per_epoch_per_rank": 138,
|
| 104 |
+
"world_size": 8,
|
| 105 |
+
"global_batch_size": 128,
|
| 106 |
+
"trainable_parameters": 104684032,
|
| 107 |
+
"parameter_groups": {
|
| 108 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 109 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 110 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 111 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 112 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 113 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 114 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 115 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 116 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 117 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 118 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 119 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 120 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 121 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 122 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 123 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 124 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 125 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 126 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 127 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 128 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 129 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 130 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 131 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 132 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 133 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 134 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 135 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 136 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 137 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 138 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 139 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 140 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 141 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 142 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 143 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 144 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 145 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 146 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 147 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 148 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 149 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 150 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 151 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 152 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 153 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 154 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 155 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 156 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 157 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 158 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 159 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 160 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 161 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 162 |
+
"residual_out.weight": "fusion_residual",
|
| 163 |
+
"residual_out.bias": "fusion_residual"
|
| 164 |
+
},
|
| 165 |
+
"activation_dtype": "bfloat16",
|
| 166 |
+
"trainable_parameter_dtype": "float32",
|
| 167 |
+
"swanlab_run_id": "4h9zkrjp"
|
| 168 |
+
}
|
| 169 |
+
{"avg_micro_time_s": 7.570369130116887, "epoch": 0, "flow_mse": 1.749738797545433, "global_step": 1, "grad_norm_max": 1.2446913719177246, "hidden_mse": 0.36647121980786324, "loss": 1.7863859087228775, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 34.36879062652588}
|
| 170 |
+
{"avg_micro_time_s": 0.9235374251277082, "epoch": 0, "flow_mse": 1.7364688764015832, "global_step": 10, "grad_norm_max": 1.646432876586914, "hidden_mse": 0.3613196772833665, "loss": 1.772600847813818, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 35.17836236953735}
|
| 171 |
+
{"avg_micro_time_s": 0.9477006714761956, "epoch": 0, "flow_mse": 1.7013526231050493, "global_step": 20, "grad_norm_max": 1.9521243572235107, "hidden_mse": 0.39329087249934674, "loss": 1.7406817123293878, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 172 |
+
{"avg_micro_time_s": 0.9182391176000237, "epoch": 0, "flow_mse": 1.5814181581139566, "global_step": 30, "grad_norm_max": 1.3988021612167358, "hidden_mse": 0.6514715321362019, "loss": 1.6465653091669084, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 173 |
+
{"avg_micro_time_s": 0.9201161204808159, "epoch": 0, "flow_mse": 1.4051025182008745, "global_step": 40, "grad_norm_max": 1.6225571632385254, "hidden_mse": 1.5282157450914384, "loss": 1.5579240918159485, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 174 |
+
{"avg_micro_time_s": 0.885716823443363, "epoch": 0, "flow_mse": 1.3408063381910325, "global_step": 50, "grad_norm_max": 1.286842703819275, "hidden_mse": 1.6829891920089723, "loss": 1.5091052532196045, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 35.17836236953735}
|
| 175 |
+
{"avg_micro_time_s": 0.9682921237908886, "epoch": 0, "flow_mse": 1.2905805960297585, "global_step": 60, "grad_norm_max": 1.5753954648971558, "hidden_mse": 1.68854421377182, "loss": 1.4594350248575212, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 176 |
+
{"avg_micro_time_s": 0.8276773064644658, "epoch": 0, "flow_mse": 1.2373953193426133, "global_step": 70, "grad_norm_max": 4.981419563293457, "hidden_mse": 1.472137273848057, "loss": 1.3846090495586396, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 177 |
+
{"avg_micro_time_s": 0.8491392163414275, "epoch": 0, "flow_mse": 1.1866303592920304, "global_step": 80, "grad_norm_max": 2.3600127696990967, "hidden_mse": 1.3034491568803788, "loss": 1.3169752791523934, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 178 |
+
{"avg_micro_time_s": 0.859149495534075, "epoch": 0, "flow_mse": 1.114801399409771, "global_step": 90, "grad_norm_max": 1.5725868940353394, "hidden_mse": 1.2420757561922073, "loss": 1.2390089735388756, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 35.17836236953735}
|
| 179 |
+
{"avg_micro_time_s": 0.9303631344650057, "epoch": 0, "flow_mse": 1.0345885261893273, "global_step": 100, "grad_norm_max": 2.7570018768310547, "hidden_mse": 1.1188017934560777, "loss": 1.1464687049388886, "lr_blocks": 1.0000000000000001e-07, "lr_fusion": 0.0001, "peak_memory_gib": 35.17836236953735}
|
| 180 |
+
{"avg_micro_time_s": 0.9938589973040508, "epoch": 0, "flow_mse": 0.9382504552602768, "global_step": 110, "grad_norm_max": 1.5205167531967163, "hidden_mse": 1.055648235976696, "loss": 1.0438152834773065, "lr_blocks": 1.1e-06, "lr_fusion": 9.999316524962345e-05, "peak_memory_gib": 35.17836236953735}
|
| 181 |
+
{"avg_micro_time_s": 1.0039036810034305, "epoch": 0, "flow_mse": 0.8337112307548523, "global_step": 120, "grad_norm_max": 2.080568790435791, "hidden_mse": 1.015562180429697, "loss": 0.935267448425293, "lr_blocks": 2.1000000000000002e-06, "lr_fusion": 9.997266286704631e-05, "peak_memory_gib": 35.17836236953735}
|
| 182 |
+
{"avg_micro_time_s": 0.7827666435216089, "epoch": 0, "flow_mse": 0.735084230452776, "global_step": 130, "grad_norm_max": 2.9411489963531494, "hidden_mse": 0.9859928995370866, "loss": 0.8336835190653802, "lr_blocks": 3.1000000000000004e-06, "lr_fusion": 9.993849845741524e-05, "peak_memory_gib": 35.17836236953735}
|
| 183 |
+
{"avg_micro_time_s": 1.022068373618822, "epoch": 1, "flow_mse": 0.650863914191723, "global_step": 140, "grad_norm_max": 2.211087465286255, "hidden_mse": 0.9673559084534645, "loss": 0.747599507123232, "lr_blocks": 4.1e-06, "lr_fusion": 9.989068136093873e-05, "peak_memory_gib": 35.17836236953735}
|
| 184 |
+
{"avg_micro_time_s": 0.9930119908167399, "epoch": 1, "flow_mse": 0.5695458181202412, "global_step": 150, "grad_norm_max": 2.5299692153930664, "hidden_mse": 0.94266607016325, "loss": 0.6638124249875546, "lr_blocks": 5.1e-06, "lr_fusion": 9.98292246503335e-05, "peak_memory_gib": 35.17836236953735}
|
| 185 |
+
{"avg_micro_time_s": 0.9784933790913783, "epoch": 1, "flow_mse": 0.484642056375742, "global_step": 160, "grad_norm_max": 2.2775866985321045, "hidden_mse": 0.9495147302746774, "loss": 0.5795935288071633, "lr_blocks": 6.1e-06, "lr_fusion": 9.975414512725057e-05, "peak_memory_gib": 35.17836236953735}
|
| 186 |
+
{"avg_micro_time_s": 0.9803913836367428, "epoch": 1, "flow_mse": 0.4089695025235415, "global_step": 170, "grad_norm_max": 2.7604339122772217, "hidden_mse": 0.9396911256015301, "loss": 0.5029386159032584, "lr_blocks": 7.100000000000001e-06, "lr_fusion": 9.966546331768191e-05, "peak_memory_gib": 35.17836236953735}
|
| 187 |
+
{"avg_micro_time_s": 0.8476272877975135, "epoch": 1, "flow_mse": 0.3592797573655844, "global_step": 180, "grad_norm_max": 2.7112045288085938, "hidden_mse": 0.9508873365819455, "loss": 0.4543684929609299, "lr_blocks": 8.1e-06, "lr_fusion": 9.956320346634876e-05, "peak_memory_gib": 35.17836236953735}
|
| 188 |
+
{"avg_micro_time_s": 0.8388726397111896, "epoch": 1, "flow_mse": 0.32551304064691067, "global_step": 190, "grad_norm_max": 3.7532150745391846, "hidden_mse": 0.937762850522995, "loss": 0.4192893292754889, "lr_blocks": 9.100000000000001e-06, "lr_fusion": 9.944739353007344e-05, "peak_memory_gib": 35.17836236953735}
|
| 189 |
+
{"avg_micro_time_s": 0.9618764713944984, "epoch": 1, "flow_mse": 0.2926826797425747, "global_step": 200, "grad_norm_max": 2.8755624294281006, "hidden_mse": 0.9361465163528919, "loss": 0.38629733547568323, "lr_blocks": 1e-05, "lr_fusion": 9.931806517013612e-05, "peak_memory_gib": 35.17836236953735}
|
| 190 |
+
{"avg_micro_time_s": 0.9065074106416433, "epoch": 1, "flow_mse": 0.269664204493165, "global_step": 210, "grad_norm_max": 1.8583863973617554, "hidden_mse": 0.903052056580782, "loss": 0.3599694143980742, "lr_blocks": 9.999238475781957e-06, "lr_fusion": 9.917525374361912e-05, "peak_memory_gib": 35.17836236953735}
|
| 191 |
+
{"avg_micro_time_s": 0.9967275719391182, "epoch": 1, "flow_mse": 0.24217076469212773, "global_step": 220, "grad_norm_max": 2.341717481613159, "hidden_mse": 0.8640865027904511, "loss": 0.32857941649854183, "lr_blocks": 9.99695413509548e-06, "lr_fusion": 9.901899829374047e-05, "peak_memory_gib": 35.17836236953735}
|
| 192 |
+
{"avg_micro_time_s": 0.8739712547787349, "epoch": 1, "flow_mse": 0.21175110191106797, "global_step": 230, "grad_norm_max": 2.7421553134918213, "hidden_mse": 0.7864921517670155, "loss": 0.29040031768381597, "lr_blocks": 9.993147673772869e-06, "lr_fusion": 9.884934153917997e-05, "peak_memory_gib": 35.17836236953735}
|
| 193 |
+
{"avg_micro_time_s": 0.8700352624873631, "epoch": 1, "flow_mse": 0.1959220325574279, "global_step": 240, "grad_norm_max": 3.033174753189087, "hidden_mse": 0.6999407097697259, "loss": 0.2659161040559411, "lr_blocks": 9.987820251299121e-06, "lr_fusion": 9.86663298624003e-05, "peak_memory_gib": 35.17836236953735}
|
| 194 |
+
{"avg_micro_time_s": 0.7937017672375077, "epoch": 1, "flow_mse": 0.17917395252734425, "global_step": 250, "grad_norm_max": 3.5113766193389893, "hidden_mse": 0.6309955686330796, "loss": 0.24227351117879153, "lr_blocks": 9.980973490458728e-06, "lr_fusion": 9.847001329696653e-05, "peak_memory_gib": 35.17836236953735}
|
| 195 |
+
{"avg_micro_time_s": 0.8086659014545149, "epoch": 1, "flow_mse": 0.16980027128010988, "global_step": 260, "grad_norm_max": 3.8903393745422363, "hidden_mse": 0.5757316552102566, "loss": 0.22737343832850457, "lr_blocks": 9.972609476841368e-06, "lr_fusion": 9.826044551386744e-05, "peak_memory_gib": 35.17836236953735}
|
| 196 |
+
{"avg_micro_time_s": 0.9954796703314059, "epoch": 1, "flow_mse": 0.1461997867561877, "global_step": 270, "grad_norm_max": 2.6041982173919678, "hidden_mse": 0.5242377843707801, "loss": 0.19862356688827276, "lr_blocks": 9.962730758206612e-06, "lr_fusion": 9.803768380684242e-05, "peak_memory_gib": 35.17836236953735}
|
| 197 |
+
{"avg_micro_time_s": 0.8525289646990132, "epoch": 2, "flow_mse": 0.1461880243383348, "global_step": 280, "grad_norm_max": 3.097048759460449, "hidden_mse": 0.48127126283943655, "loss": 0.1943151516839862, "lr_blocks": 9.951340343707852e-06, "lr_fusion": 9.780178907671789e-05, "peak_memory_gib": 35.17836236953735}
|
| 198 |
+
{"avg_micro_time_s": 0.9279171040907386, "epoch": 2, "flow_mse": 0.13018152639269828, "global_step": 290, "grad_norm_max": 3.6525473594665527, "hidden_mse": 0.4434080045670271, "loss": 0.17452232725918293, "lr_blocks": 9.938441702975689e-06, "lr_fusion": 9.755282581475769e-05, "peak_memory_gib": 35.17836236953735}
|
| 199 |
+
{"avg_micro_time_s": 0.8557695019393577, "epoch": 2, "flow_mse": 0.12391671296209097, "global_step": 300, "grad_norm_max": 2.665829658508301, "hidden_mse": 0.40752875953912737, "loss": 0.1646695898845792, "lr_blocks": 9.924038765061042e-06, "lr_fusion": 9.729086208503174e-05, "peak_memory_gib": 35.17836236953735}
|
| 200 |
+
{"avg_micro_time_s": 0.8965211596267182, "epoch": 2, "flow_mse": 0.12429976435378194, "global_step": 310, "grad_norm_max": 4.673442840576172, "hidden_mse": 0.38589291982352736, "loss": 0.1628890570253134, "lr_blocks": 9.908135917238321e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 35.17836236953735}
|
| 201 |
+
{"avg_micro_time_s": 0.9538184367207578, "epoch": 2, "flow_mse": 0.12174742873758078, "global_step": 320, "grad_norm_max": 3.8317954540252686, "hidden_mse": 0.3710258278995753, "loss": 0.15885001290589573, "lr_blocks": 9.890738003669029e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 35.17836236953735}
|
| 202 |
+
W0725 13:34:33.083000 3351082 site-packages/torch/distributed/elastic/agent/server/api.py:704] Received Signals.SIGTERM death signal, shutting down workers
|
| 203 |
+
W0725 13:34:33.084000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351215 closing signal SIGTERM
|
| 204 |
+
W0725 13:34:33.085000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351216 closing signal SIGTERM
|
| 205 |
+
W0725 13:34:33.087000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351217 closing signal SIGTERM
|
| 206 |
+
W0725 13:34:33.090000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351218 closing signal SIGTERM
|
| 207 |
+
W0725 13:34:33.092000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351219 closing signal SIGTERM
|
| 208 |
+
W0725 13:34:33.095000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351220 closing signal SIGTERM
|
| 209 |
+
W0725 13:34:33.096000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351221 closing signal SIGTERM
|
| 210 |
+
W0725 13:34:33.097000 3351082 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3351222 closing signal SIGTERM
|
| 211 |
+
Traceback (most recent call last):
|
| 212 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/runpy.py", line 196, in _run_module_as_main
|
| 213 |
+
return _run_code(code, main_globals, None,
|
| 214 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/runpy.py", line 86, in _run_code
|
| 215 |
+
exec(code, run_globals)
|
| 216 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 923, in <module>
|
| 217 |
+
main()
|
| 218 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 219 |
+
return f(*args, **kwargs)
|
| 220 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 919, in main
|
| 221 |
+
run(args)
|
| 222 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 910, in run
|
| 223 |
+
elastic_launch(
|
| 224 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__
|
| 225 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 226 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent
|
| 227 |
+
result = agent.run()
|
| 228 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper
|
| 229 |
+
result = f(*args, **kwargs)
|
| 230 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 696, in run
|
| 231 |
+
result = self._invoke_run(role)
|
| 232 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 855, in _invoke_run
|
| 233 |
+
time.sleep(monitor_interval)
|
| 234 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 235 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 236 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 3351082 got signal: 15
|
| 237 |
+
[2026-07-25T13:35:17Z] 从完整 NVMe 数据启动 8 GPU Predictor-v4 训练。
|
| 238 |
+
WARNING:__main__:
|
| 239 |
+
*****************************************
|
| 240 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 241 |
+
*****************************************
|
| 242 |
+
[rank6]:[W725 13:35:30.626946342 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 243 |
+
[rank3]:[W725 13:35:31.073151716 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 244 |
+
[rank1]:[W725 13:35:31.260400994 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 245 |
+
[rank5]:[W725 13:35:31.300441880 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 246 |
+
[rank0]:[W725 13:35:31.309708448 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 247 |
+
[rank2]:[W725 13:35:31.342651553 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 248 |
+
[rank4]:[W725 13:35:31.349925878 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 249 |
+
[rank7]:[W725 13:35:31.356336128 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 250 |
+
swanlab: Currently logged in as: ZhuoChen to https://api.swanlab.cn. Use
|
| 251 |
+
`swanlab login --relogin` to force relogin
|
| 252 |
+
swanlab: Tracking run with swanlab version 0.9.0
|
| 253 |
+
swanlab: 💾 Run data saved at
|
| 254 |
+
/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/swanlab/
|
| 255 |
+
run-20260725_133617-zd7a5ge6
|
| 256 |
+
swanlab: 📁 View project at
|
| 257 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4
|
| 258 |
+
swanlab: 🚀 View run pre...1-28-prompts1000-seed0-bs32x8-resume300 at
|
| 259 |
+
https://swanlab.cn/@ZhuoChen/Self-Forcing-Predictor-v4/runs/zd7a5ge6
|
| 260 |
+
{"swanlab_run_id": "zd7a5ge6"}
|
| 261 |
+
{
|
| 262 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 263 |
+
"prompt_count": 1000,
|
| 264 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 265 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 266 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 267 |
+
"max_records": null,
|
| 268 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 269 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 270 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 271 |
+
"strict_teacher_load": true,
|
| 272 |
+
"source_block_ids": [
|
| 273 |
+
1,
|
| 274 |
+
28
|
| 275 |
+
],
|
| 276 |
+
"timestep_shift": 5.0,
|
| 277 |
+
"local_attn_size": -1,
|
| 278 |
+
"sink_size": 0,
|
| 279 |
+
"latent_height": 60,
|
| 280 |
+
"latent_width": 104,
|
| 281 |
+
"max_steps": 2000,
|
| 282 |
+
"batch_size": 32,
|
| 283 |
+
"gradient_accumulation_steps": 1,
|
| 284 |
+
"num_workers": 0,
|
| 285 |
+
"pin_memory": false,
|
| 286 |
+
"persistent_workers": false,
|
| 287 |
+
"fusion_lr": 0.0001,
|
| 288 |
+
"blocks_lr": 1e-05,
|
| 289 |
+
"beta1": 0.9,
|
| 290 |
+
"beta2": 0.95,
|
| 291 |
+
"weight_decay": 0.01,
|
| 292 |
+
"scheduler_warmup_steps": 100,
|
| 293 |
+
"block_warmup_steps": 100,
|
| 294 |
+
"hidden_loss_weight": 0.1,
|
| 295 |
+
"flow_loss_weight": 1.0,
|
| 296 |
+
"grad_clip": 1.0,
|
| 297 |
+
"fp32_trainable_params": true,
|
| 298 |
+
"gradient_checkpointing": false,
|
| 299 |
+
"find_unused_parameters": false,
|
| 300 |
+
"log_every": 10,
|
| 301 |
+
"save_every": 100,
|
| 302 |
+
"keep_snapshots": 20,
|
| 303 |
+
"resume": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/training_latest.pt",
|
| 304 |
+
"use_wandb": false,
|
| 305 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 306 |
+
"wandb_entity": null,
|
| 307 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 308 |
+
"use_swanlab": true,
|
| 309 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 310 |
+
"swanlab_workspace": null,
|
| 311 |
+
"swanlab_experiment": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0-bs32x8-resume300",
|
| 312 |
+
"swanlab_description": "Self-Forcing Wan2.1 1.3B Predictor-v4 offline prefeature training",
|
| 313 |
+
"swanlab_tags": [
|
| 314 |
+
"predictor-v4",
|
| 315 |
+
"self-forcing",
|
| 316 |
+
"wan2.1-1.3b",
|
| 317 |
+
"offline-prefeature",
|
| 318 |
+
"blocks-1-28",
|
| 319 |
+
"8gpu",
|
| 320 |
+
"seed0"
|
| 321 |
+
],
|
| 322 |
+
"swanlab_mode": "cloud",
|
| 323 |
+
"supervision_pairs": [
|
| 324 |
+
[
|
| 325 |
+
0,
|
| 326 |
+
1
|
| 327 |
+
],
|
| 328 |
+
[
|
| 329 |
+
1,
|
| 330 |
+
2
|
| 331 |
+
],
|
| 332 |
+
[
|
| 333 |
+
2,
|
| 334 |
+
3
|
| 335 |
+
]
|
| 336 |
+
],
|
| 337 |
+
"manifest_records": 6000,
|
| 338 |
+
"dataset_pairs": 18000,
|
| 339 |
+
"steps_per_epoch_per_rank": 66,
|
| 340 |
+
"world_size": 8,
|
| 341 |
+
"global_batch_size": 256,
|
| 342 |
+
"trainable_parameters": 104684032,
|
| 343 |
+
"parameter_groups": {
|
| 344 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 345 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 346 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 347 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 348 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 349 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 350 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 351 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 352 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 353 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 354 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 355 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 356 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 357 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 358 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 359 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 360 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 361 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 362 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 363 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 364 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 365 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 366 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 367 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 368 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 369 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 370 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 371 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 372 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 373 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 374 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 375 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 376 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 377 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 378 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 379 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 380 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 381 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 382 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 383 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 384 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 385 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 386 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 387 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 388 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 389 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 390 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 391 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 392 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 393 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 394 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 395 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 396 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 397 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 398 |
+
"residual_out.weight": "fusion_residual",
|
| 399 |
+
"residual_out.bias": "fusion_residual"
|
| 400 |
+
},
|
| 401 |
+
"activation_dtype": "bfloat16",
|
| 402 |
+
"trainable_parameter_dtype": "float32",
|
| 403 |
+
"swanlab_run_id": "zd7a5ge6"
|
| 404 |
+
}
|
| 405 |
+
{"avg_micro_time_s": 2.629857139934029, "epoch": 2, "flow_mse": 0.12393017988651991, "global_step": 310, "grad_norm_max": 3.2882611751556396, "hidden_mse": 0.38931699469685555, "loss": 0.16286188066005708, "lr_blocks": 9.908135917238321e-06, "lr_fusion": 9.701596950580806e-05, "peak_memory_gib": 68.46308755874634}
|
| 406 |
+
{"avg_micro_time_s": 1.949286403792212, "epoch": 2, "flow_mse": 0.11613796576857567, "global_step": 320, "grad_norm_max": 2.581794500350952, "hidden_mse": 0.37093467339873315, "loss": 0.1532314345240593, "lr_blocks": 9.890738003669029e-06, "lr_fusion": 9.672822322997305e-05, "peak_memory_gib": 68.46350812911987}
|
| 407 |
+
{"avg_micro_time_s": 1.9139812438632362, "epoch": 2, "flow_mse": 0.11843711538240315, "global_step": 330, "grad_norm_max": 2.837050437927246, "hidden_mse": 0.3591951116919518, "loss": 0.15435662725940347, "lr_blocks": 9.871850323926178e-06, "lr_fusion": 9.642770192448536e-05, "peak_memory_gib": 68.46350812911987}
|
| 408 |
+
{"avg_micro_time_s": 1.9638592657793197, "epoch": 2, "flow_mse": 0.10847121011465788, "global_step": 340, "grad_norm_max": 3.7426559925079346, "hidden_mse": 0.33858932666480546, "loss": 0.14233014369383454, "lr_blocks": 9.851478631379982e-06, "lr_fusion": 9.611448774886924e-05, "peak_memory_gib": 68.46350812911987}
|
| 409 |
+
{"avg_micro_time_s": 1.9809167194078328, "epoch": 3, "flow_mse": 0.1031073421239853, "global_step": 350, "grad_norm_max": 2.80180025100708, "hidden_mse": 0.32489884421229365, "loss": 0.13559722723439335, "lr_blocks": 9.829629131445342e-06, "lr_fusion": 9.578866633275288e-05, "peak_memory_gib": 68.46350812911987}
|
| 410 |
+
{"avg_micro_time_s": 1.790873375805677, "epoch": 3, "flow_mse": 0.10435326369479299, "global_step": 360, "grad_norm_max": 3.61527681350708, "hidden_mse": 0.31754680760204795, "loss": 0.1361079447902739, "lr_blocks": 9.806308479691595e-06, "lr_fusion": 9.545032675245813e-05, "peak_memory_gib": 68.46350812911987}
|
| 411 |
+
{"avg_micro_time_s": 1.6745291174505839, "epoch": 3, "flow_mse": 0.10100243939086795, "global_step": 370, "grad_norm_max": 3.1019058227539062, "hidden_mse": 0.30805569849908354, "loss": 0.13180800881236793, "lr_blocks": 9.781523779815178e-06, "lr_fusion": 9.509956150664796e-05, "peak_memory_gib": 68.46350812911987}
|
| 412 |
+
{"avg_micro_time_s": 1.6829090670857114, "epoch": 3, "flow_mse": 0.09974822653457523, "global_step": 380, "grad_norm_max": 4.146679401397705, "hidden_mse": 0.30044727586209774, "loss": 0.1297929552383721, "lr_blocks": 9.755282581475769e-06, "lr_fusion": 9.473646649103818e-05, "peak_memory_gib": 68.46350812911987}
|
| 413 |
+
{"avg_micro_time_s": 1.876903708356258, "epoch": 3, "flow_mse": 0.10485911099240185, "global_step": 390, "grad_norm_max": 1.9585886001586914, "hidden_mse": 0.29475032463669776, "loss": 0.13433414353057743, "lr_blocks": 9.727592877996585e-06, "lr_fusion": 9.43611409721806e-05, "peak_memory_gib": 68.46350812911987}
|
| 414 |
+
{"avg_micro_time_s": 1.9579433922859608, "epoch": 3, "flow_mse": 0.09907718747854233, "global_step": 400, "grad_norm_max": 3.6999549865722656, "hidden_mse": 0.2881328942254186, "loss": 0.12789047760888936, "lr_blocks": 9.698463103929542e-06, "lr_fusion": 9.397368756032445e-05, "peak_memory_gib": 68.46350812911987}
|
| 415 |
+
{"avg_micro_time_s": 1.83207836826914, "epoch": 4, "flow_mse": 0.10220368318259716, "global_step": 410, "grad_norm_max": 2.5350282192230225, "hidden_mse": 0.28303657546639444, "loss": 0.13050734112039208, "lr_blocks": 9.667902132486009e-06, "lr_fusion": 9.357421218136386e-05, "peak_memory_gib": 68.46350812911987}
|
| 416 |
+
{"avg_micro_time_s": 1.769808326018392, "epoch": 4, "flow_mse": 0.1039027538150549, "global_step": 420, "grad_norm_max": 2.6458826065063477, "hidden_mse": 0.2822224203497172, "loss": 0.1321249960921705, "lr_blocks": 9.635919272833938e-06, "lr_fusion": 9.316282404787871e-05, "peak_memory_gib": 68.46350812911987}
|
| 417 |
+
{"avg_micro_time_s": 1.8770967723452487, "epoch": 4, "flow_mse": 0.09036883087828756, "global_step": 430, "grad_norm_max": 2.7844719886779785, "hidden_mse": 0.2689092235639691, "loss": 0.11725975405424834, "lr_blocks": 9.602524267262202e-06, "lr_fusion": 9.273963562927695e-05, "peak_memory_gib": 68.46350812911987}
|
| 418 |
+
{"avg_micro_time_s": 1.9124310042083381, "epoch": 4, "flow_mse": 0.09349321946501732, "global_step": 440, "grad_norm_max": 2.967038154602051, "hidden_mse": 0.2642635779455304, "loss": 0.11991957789286972, "lr_blocks": 9.567727288213005e-06, "lr_fusion": 9.230476262104677e-05, "peak_memory_gib": 68.46350812911987}
|
| 419 |
+
{"avg_micro_time_s": 1.7230479505989935, "epoch": 4, "flow_mse": 0.09501845473423601, "global_step": 450, "grad_norm_max": 2.671290397644043, "hidden_mse": 0.259578475728631, "loss": 0.12097630277276039, "lr_blocks": 9.531538935183252e-06, "lr_fusion": 9.185832391312644e-05, "peak_memory_gib": 68.46350812911987}
|
| 420 |
+
{"avg_micro_time_s": 1.8449677960408737, "epoch": 4, "flow_mse": 0.088533672131598, "global_step": 460, "grad_norm_max": 2.018291711807251, "hidden_mse": 0.2519702633842826, "loss": 0.11373069928959012, "lr_blocks": 9.493970231495836e-06, "lr_fusion": 9.140044155740101e-05, "peak_memory_gib": 68.46350812911987}
|
| 421 |
+
{"avg_micro_time_s": 2.000304789120855, "epoch": 4, "flow_mse": 0.0845537671353668, "global_step": 470, "grad_norm_max": 2.4637300968170166, "hidden_mse": 0.25105361733585596, "loss": 0.10965913012623787, "lr_blocks": 9.45503262094184e-06, "lr_fusion": 9.093124073433463e-05, "peak_memory_gib": 68.46350812911987}
|
| 422 |
+
{"avg_micro_time_s": 1.771543370553991, "epoch": 5, "flow_mse": 0.0848984924145043, "global_step": 480, "grad_norm_max": 2.370950222015381, "hidden_mse": 0.24045866541564465, "loss": 0.10894435914233327, "lr_blocks": 9.414737964294636e-06, "lr_fusion": 9.045084971874738e-05, "peak_memory_gib": 68.46350812911987}
|
| 423 |
+
{"avg_micro_time_s": 1.8954340051379406, "epoch": 5, "flow_mse": 0.08467793129384518, "global_step": 490, "grad_norm_max": 2.4222309589385986, "hidden_mse": 0.23431672640144827, "loss": 0.10810960466042162, "lr_blocks": 9.37309853569698e-06, "lr_fusion": 8.995939984474624e-05, "peak_memory_gib": 68.46350812911987}
|
| 424 |
+
{"avg_micro_time_s": 1.8990013301940054, "epoch": 5, "flow_mse": 0.08086096784099937, "global_step": 500, "grad_norm_max": 2.5167181491851807, "hidden_mse": 0.22931088581681253, "loss": 0.10379205718636514, "lr_blocks": 9.330127018922195e-06, "lr_fusion": 8.945702546981969e-05, "peak_memory_gib": 68.46350812911987}
|
| 425 |
+
{"avg_micro_time_s": 1.857530340390804, "epoch": 5, "flow_mse": 0.07947712298482656, "global_step": 510, "grad_norm_max": 1.5499550104141235, "hidden_mse": 0.22288411129266025, "loss": 0.10176553511992098, "lr_blocks": 9.285836503510562e-06, "lr_fusion": 8.894386393810563e-05, "peak_memory_gib": 68.46350812911987}
|
| 426 |
+
{"avg_micro_time_s": 1.8259927214239724, "epoch": 5, "flow_mse": 0.08615787411108614, "global_step": 520, "grad_norm_max": 2.1541709899902344, "hidden_mse": 0.2208554781973362, "loss": 0.10824342193081976, "lr_blocks": 9.24024048078213e-06, "lr_fusion": 8.842005554284296e-05, "peak_memory_gib": 68.46350812911987}
|
| 427 |
+
{"avg_micro_time_s": 1.850162015359092, "epoch": 5, "flow_mse": 0.08251581969670951, "global_step": 530, "grad_norm_max": 2.1317837238311768, "hidden_mse": 0.2161341816186905, "loss": 0.10412923851981759, "lr_blocks": 9.193352839727122e-06, "lr_fusion": 8.788574348801675e-05, "peak_memory_gib": 68.46350812911987}
|
| 428 |
+
{"avg_micro_time_s": 1.7231032210023842, "epoch": 5, "flow_mse": 0.07909762635827065, "global_step": 540, "grad_norm_max": 2.1987521648406982, "hidden_mse": 0.21046547647565605, "loss": 0.10014417488127947, "lr_blocks": 9.145187862775208e-06, "lr_fusion": 8.73410738492077e-05, "peak_memory_gib": 68.46350812911987}
|
| 429 |
+
{"avg_micro_time_s": 1.8184389490634203, "epoch": 6, "flow_mse": 0.08191823242232205, "global_step": 550, "grad_norm_max": 1.9033032655715942, "hidden_mse": 0.21054420229047538, "loss": 0.10297265322878957, "lr_blocks": 9.09576022144496e-06, "lr_fusion": 8.678619553365659e-05, "peak_memory_gib": 68.46350812911987}
|
| 430 |
+
{"avg_micro_time_s": 1.8944095356826438, "epoch": 6, "flow_mse": 0.07599904504604638, "global_step": 560, "grad_norm_max": 1.926156759262085, "hidden_mse": 0.20284319352358582, "loss": 0.09628336504101753, "lr_blocks": 9.045084971874738e-06, "lr_fusion": 8.622126023955446e-05, "peak_memory_gib": 68.46350812911987}
|
| 431 |
+
{"avg_micro_time_s": 1.6680899022234372, "epoch": 6, "flow_mse": 0.07714098119176925, "global_step": 570, "grad_norm_max": 2.8186357021331787, "hidden_mse": 0.20087382700294257, "loss": 0.09722836464643479, "lr_blocks": 8.993177550236464e-06, "lr_fusion": 8.564642241456986e-05, "peak_memory_gib": 68.46350812911987}
|
| 432 |
+
{"avg_micro_time_s": 1.840931454865495, "epoch": 6, "flow_mse": 0.07506008534692228, "global_step": 580, "grad_norm_max": 2.2306644916534424, "hidden_mse": 0.19238721020519733, "loss": 0.09429880687966943, "lr_blocks": 8.94005376803361e-06, "lr_fusion": 8.506183921362443e-05, "peak_memory_gib": 68.46350812911987}
|
| 433 |
+
{"avg_micro_time_s": 1.9420451847268851, "epoch": 6, "flow_mse": 0.07425902322866022, "global_step": 590, "grad_norm_max": 2.5445029735565186, "hidden_mse": 0.18635293375700712, "loss": 0.09289431720972062, "lr_blocks": 8.885729807284855e-06, "lr_fusion": 8.44676704559283e-05, "peak_memory_gib": 68.46350812911987}
|
| 434 |
+
{"avg_micro_time_s": 1.878620579345443, "epoch": 6, "flow_mse": 0.0771529158577323, "global_step": 600, "grad_norm_max": 1.5593407154083252, "hidden_mse": 0.18661604821681976, "loss": 0.09581452133134008, "lr_blocks": 8.83022221559489e-06, "lr_fusion": 8.386407858128706e-05, "peak_memory_gib": 68.46350812911987}
|
| 435 |
+
{"avg_micro_time_s": 1.7841684922925198, "epoch": 7, "flow_mse": 0.07418268383480608, "global_step": 610, "grad_norm_max": 1.844347596168518, "hidden_mse": 0.18209756892174483, "loss": 0.09239244163036348, "lr_blocks": 8.773547901113862e-06, "lr_fusion": 8.32512286056924e-05, "peak_memory_gib": 68.46350812911987}
|
| 436 |
+
{"avg_micro_time_s": 1.7495355686536642, "epoch": 7, "flow_mse": 0.07441529156640173, "global_step": 620, "grad_norm_max": 2.013097047805786, "hidden_mse": 0.1817359069362283, "loss": 0.0925888827070594, "lr_blocks": 8.715724127386971e-06, "lr_fusion": 8.262928807620843e-05, "peak_memory_gib": 68.46350812911987}
|
| 437 |
+
2026-07-25 14:13:21.019 | ERROR |
|
| 438 |
+
swanlab.sdk.internal.pkg.client.session:send:131 - [HTTP] POST
|
| 439 |
+
https://api.swanlab.cn/api/house/metrics -> 522 (5099ms) trace:unknown | [ERR]
|
| 440 |
+
code=unknown code message=unknown error
|
| 441 |
+
2026-07-25 14:13:23.554 | WARNING |
|
| 442 |
+
swanlab.sdk.internal.core_python.transport.thread:warn:187 - Upload failed due
|
| 443 |
+
to network or server issues, retrying automatically.
|
| 444 |
+
swanlab: Upload recovered, resuming normally.
|
| 445 |
+
{"avg_micro_time_s": 2.0745983355402133, "epoch": 7, "flow_mse": 0.07575193946249784, "global_step": 630, "grad_norm_max": 1.4495962858200073, "hidden_mse": 0.17780817206948996, "loss": 0.09353275746107102, "lr_blocks": 8.656768508095853e-06, "lr_fusion": 8.199842702516583e-05, "peak_memory_gib": 68.46350812911987}
|
| 446 |
+
{"avg_micro_time_s": 1.6178329646398197, "epoch": 7, "flow_mse": 0.07271728334017098, "global_step": 640, "grad_norm_max": 2.1036791801452637, "hidden_mse": 0.1734720304608345, "loss": 0.09006448723375798, "lr_blocks": 8.596699001693257e-06, "lr_fusion": 8.135881792367686e-05, "peak_memory_gib": 68.46350812911987}
|
| 447 |
+
{"avg_micro_time_s": 1.8334216722782004, "epoch": 7, "flow_mse": 0.07158588757738471, "global_step": 650, "grad_norm_max": 1.806536078453064, "hidden_mse": 0.16676762849092486, "loss": 0.08826265102252365, "lr_blocks": 8.535533905932739e-06, "lr_fusion": 8.07106356344834e-05, "peak_memory_gib": 68.46350812911987}
|
| 448 |
+
{"avg_micro_time_s": 1.9290761913187453, "epoch": 7, "flow_mse": 0.07200726652517915, "global_step": 660, "grad_norm_max": 1.3992115259170532, "hidden_mse": 0.16250971648842097, "loss": 0.08825823860242964, "lr_blocks": 8.473291852294986e-06, "lr_fusion": 8.005405736415126e-05, "peak_memory_gib": 68.46350812911987}
|
| 449 |
+
{"avg_micro_time_s": 1.8624601541116137, "epoch": 7, "flow_mse": 0.07254991228692234, "global_step": 670, "grad_norm_max": 1.6715388298034668, "hidden_mse": 0.16857999172061683, "loss": 0.08940791198983788, "lr_blocks": 8.409991800312493e-06, "lr_fusion": 7.938926261462366e-05, "peak_memory_gib": 68.46350812911987}
|
| 450 |
+
{"avg_micro_time_s": 1.7186719533419819, "epoch": 8, "flow_mse": 0.06775221680290998, "global_step": 680, "grad_norm_max": 2.1073012351989746, "hidden_mse": 0.1592436818405986, "loss": 0.08367658536881209, "lr_blocks": 8.345653031794292e-06, "lr_fusion": 7.871643313414718e-05, "peak_memory_gib": 68.46350812911987}
|
| 451 |
+
{"avg_micro_time_s": 2.054523778187286, "epoch": 8, "flow_mse": 0.06942765288986266, "global_step": 690, "grad_norm_max": 1.4319833517074585, "hidden_mse": 0.15960346888750793, "loss": 0.08538800021633507, "lr_blocks": 8.280295144952537e-06, "lr_fusion": 7.803575286758364e-05, "peak_memory_gib": 68.46350812911987}
|
| 452 |
+
{"avg_micro_time_s": 1.7629200713461615, "epoch": 8, "flow_mse": 0.06661824253387749, "global_step": 700, "grad_norm_max": 1.8629697561264038, "hidden_mse": 0.15122736440971496, "loss": 0.08174097924493254, "lr_blocks": 8.213938048432697e-06, "lr_fusion": 7.734740790612136e-05, "peak_memory_gib": 68.46350812911987}
|
| 453 |
+
{"avg_micro_time_s": 1.765262425330002, "epoch": 8, "flow_mse": 0.07106665275059641, "global_step": 710, "grad_norm_max": 1.6708544492721558, "hidden_mse": 0.155658970028162, "loss": 0.08663254976272583, "lr_blocks": 8.146601955249187e-06, "lr_fusion": 7.66515864363997e-05, "peak_memory_gib": 68.46350812911987}
|
| 454 |
+
{"avg_micro_time_s": 2.0210482663824223, "epoch": 8, "flow_mse": 0.06746772066690028, "global_step": 720, "grad_norm_max": 1.1817511320114136, "hidden_mse": 0.150514136813581, "loss": 0.08251913487911225, "lr_blocks": 8.078307376628292e-06, "lr_fusion": 7.594847868906076e-05, "peak_memory_gib": 68.46350812911987}
|
| 455 |
+
{"avg_micro_time_s": 1.8907386586259236, "epoch": 8, "flow_mse": 0.06803802275098861, "global_step": 730, "grad_norm_max": 1.369581937789917, "hidden_mse": 0.14907391220331193, "loss": 0.08294541416689755, "lr_blocks": 8.009075115760243e-06, "lr_fusion": 7.52382768867422e-05, "peak_memory_gib": 68.46350812911987}
|
| 456 |
+
{"avg_micro_time_s": 1.6752641557410244, "epoch": 9, "flow_mse": 0.06919461712241173, "global_step": 740, "grad_norm_max": 1.4564262628555298, "hidden_mse": 0.14728272343054413, "loss": 0.08392288968898357, "lr_blocks": 7.938926261462366e-06, "lr_fusion": 7.452117519152542e-05, "peak_memory_gib": 68.46350812911987}
|
| 457 |
+
{"avg_micro_time_s": 1.7529244130506414, "epoch": 9, "flow_mse": 0.06664506304077804, "global_step": 750, "grad_norm_max": 1.4723470211029053, "hidden_mse": 0.14383591981604696, "loss": 0.08102865526452661, "lr_blocks": 7.86788218175523e-06, "lr_fusion": 7.379736965185368e-05, "peak_memory_gib": 68.46350812911987}
|
| 458 |
+
{"avg_micro_time_s": 1.744390674715396, "epoch": 9, "flow_mse": 0.06827977779321373, "global_step": 760, "grad_norm_max": 1.7442106008529663, "hidden_mse": 0.1432000114582479, "loss": 0.08259977912530303, "lr_blocks": 7.795964517353734e-06, "lr_fusion": 7.30670581489344e-05, "peak_memory_gib": 68.46350812911987}
|
| 459 |
+
{"avg_micro_time_s": 1.9586176592012636, "epoch": 9, "flow_mse": 0.06658509261906147, "global_step": 770, "grad_norm_max": 2.022984504699707, "hidden_mse": 0.1376851993612945, "loss": 0.08035361249931157, "lr_blocks": 7.723195175075136e-06, "lr_fusion": 7.233044034264034e-05, "peak_memory_gib": 68.46350812911987}
|
| 460 |
+
{"avg_micro_time_s": 1.9555875910707983, "epoch": 9, "flow_mse": 0.06847756509669126, "global_step": 780, "grad_norm_max": 1.5319805145263672, "hidden_mse": 0.13906292710453272, "loss": 0.08238385845907033, "lr_blocks": 7.649596321166024e-06, "lr_fusion": 7.158771761692464e-05, "peak_memory_gib": 68.46350812911987}
|
| 461 |
+
{"avg_micro_time_s": 1.8404817442118657, "epoch": 9, "flow_mse": 0.06554289055056871, "global_step": 790, "grad_norm_max": 1.3032727241516113, "hidden_mse": 0.13789730444550516, "loss": 0.07933262106962502, "lr_blocks": 7.575190374550272e-06, "lr_fusion": 7.083909302476453e-05, "peak_memory_gib": 68.46350812911987}
|
| 462 |
+
{"avg_micro_time_s": 1.941538252151804, "epoch": 9, "flow_mse": 0.0677900274284184, "global_step": 800, "grad_norm_max": 1.0706440210342407, "hidden_mse": 0.1393029022961855, "loss": 0.08172031748108566, "lr_blocks": 7.500000000000001e-06, "lr_fusion": 7.008477123264848e-05, "peak_memory_gib": 68.46350812911987}
|
| 463 |
+
{"avg_micro_time_s": 1.8403584161540494, "epoch": 10, "flow_mse": 0.06389061664231122, "global_step": 810, "grad_norm_max": 1.2349841594696045, "hidden_mse": 0.13415419561788441, "loss": 0.07730603613890707, "lr_blocks": 7.424048101231687e-06, "lr_fusion": 6.932495846462261e-05, "peak_memory_gib": 68.46350812911987}
|
| 464 |
+
{"avg_micro_time_s": 2.066303575213533, "epoch": 10, "flow_mse": 0.06629212941043079, "global_step": 820, "grad_norm_max": 1.2726998329162598, "hidden_mse": 0.13719576336443426, "loss": 0.08001170596107841, "lr_blocks": 7.347357813929455e-06, "lr_fusion": 6.855986244591104e-05, "peak_memory_gib": 68.46350812911987}
|
| 465 |
+
{"avg_micro_time_s": 1.8050329798075837, "epoch": 10, "flow_mse": 0.06530892113223673, "global_step": 830, "grad_norm_max": 1.4997843503952026, "hidden_mse": 0.13179729050025343, "loss": 0.07848865031264723, "lr_blocks": 7.269952498697734e-06, "lr_fusion": 6.778969234612584e-05, "peak_memory_gib": 68.46350812911987}
|
| 466 |
+
{"avg_micro_time_s": 1.8132088849451975, "epoch": 10, "flow_mse": 0.0663258790038526, "global_step": 840, "grad_norm_max": 2.0354981422424316, "hidden_mse": 0.13041680296882988, "loss": 0.0793675592634827, "lr_blocks": 7.191855733945388e-06, "lr_fusion": 6.701465872208216e-05, "peak_memory_gib": 68.46350812911987}
|
| 467 |
+
{"avg_micro_time_s": 1.8288670425405145, "epoch": 10, "flow_mse": 0.06910035191103817, "global_step": 850, "grad_norm_max": 1.2372161149978638, "hidden_mse": 0.13194104889407754, "loss": 0.08229445680044592, "lr_blocks": 7.113091308703498e-06, "lr_fusion": 6.623497346023418e-05, "peak_memory_gib": 68.46350812911987}
|
| 468 |
+
{"avg_micro_time_s": 1.9691673126260867, "epoch": 10, "flow_mse": 0.061420508706942205, "global_step": 860, "grad_norm_max": 1.1865767240524292, "hidden_mse": 0.12501905672252178, "loss": 0.073922414239496, "lr_blocks": 7.033683215379002e-06, "lr_fusion": 6.545084971874738e-05, "peak_memory_gib": 68.46350812911987}
|
| 469 |
+
{"avg_micro_time_s": 1.5611071650782833, "epoch": 10, "flow_mse": 0.06397015312686563, "global_step": 870, "grad_norm_max": 2.0459511280059814, "hidden_mse": 0.12757008243352175, "loss": 0.0767271613702178, "lr_blocks": 6.953655642446368e-06, "lr_fusion": 6.466250186922325e-05, "peak_memory_gib": 68.46350812911987}
|
| 470 |
+
W0725 14:40:22.223000 3358924 site-packages/torch/distributed/elastic/agent/server/api.py:704] Received Signals.SIGTERM death signal, shutting down workers
|
| 471 |
+
W0725 14:40:22.230000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359040 closing signal SIGTERM
|
| 472 |
+
W0725 14:40:22.235000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359041 closing signal SIGTERM
|
| 473 |
+
W0725 14:40:22.236000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359042 closing signal SIGTERM
|
| 474 |
+
W0725 14:40:22.239000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359043 closing signal SIGTERM
|
| 475 |
+
W0725 14:40:22.241000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359044 closing signal SIGTERM
|
| 476 |
+
W0725 14:40:22.244000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359045 closing signal SIGTERM
|
| 477 |
+
W0725 14:40:22.245000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359046 closing signal SIGTERM
|
| 478 |
+
W0725 14:40:22.246000 3358924 site-packages/torch/distributed/elastic/multiprocessing/api.py:897] Sending process 3359047 closing signal SIGTERM
|
| 479 |
+
Traceback (most recent call last):
|
| 480 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/runpy.py", line 196, in _run_module_as_main
|
| 481 |
+
return _run_code(code, main_globals, None,
|
| 482 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/runpy.py", line 86, in _run_code
|
| 483 |
+
exec(code, run_globals)
|
| 484 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 923, in <module>
|
| 485 |
+
main()
|
| 486 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
| 487 |
+
return f(*args, **kwargs)
|
| 488 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 919, in main
|
| 489 |
+
run(args)
|
| 490 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/run.py", line 910, in run
|
| 491 |
+
elastic_launch(
|
| 492 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 138, in __call__
|
| 493 |
+
return launch_agent(self._config, self._entrypoint, list(args))
|
| 494 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 260, in launch_agent
|
| 495 |
+
result = agent.run()
|
| 496 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper
|
| 497 |
+
result = f(*args, **kwargs)
|
| 498 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 696, in run
|
| 499 |
+
result = self._invoke_run(role)
|
| 500 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/agent/server/api.py", line 855, in _invoke_run
|
| 501 |
+
time.sleep(monitor_interval)
|
| 502 |
+
File "/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
|
| 503 |
+
raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
|
| 504 |
+
torch.distributed.elastic.multiprocessing.api.SignalException: Process 3358924 got signal: 15
|
checkpoints/predictor_v4_1000_seed0/training_and_sync_stopped_step90.log
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[2026-07-24T14:19:45Z] 从完整 NVMe 数据启动 8 GPU Predictor-v4 训练。
|
| 2 |
+
WARNING:__main__:
|
| 3 |
+
*****************************************
|
| 4 |
+
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
|
| 5 |
+
*****************************************
|
| 6 |
+
[rank2]:[W724 14:20:01.288822487 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 7 |
+
[rank6]:[W724 14:20:02.954115481 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 8 |
+
[rank3]:[W724 14:20:02.955989052 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 9 |
+
[rank7]:[W724 14:20:02.973490572 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 10 |
+
[rank4]:[W724 14:20:02.977474637 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 11 |
+
[rank0]:[W724 14:20:02.985108344 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 12 |
+
[rank1]:[W724 14:20:02.994097183 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 13 |
+
[rank5]:[W724 14:20:02.994370788 ProcessGroupNCCL.cpp:4115] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device,or call init_process_group() with a device_id.
|
| 14 |
+
{
|
| 15 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 16 |
+
"prompt_count": 1000,
|
| 17 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 18 |
+
"manifest": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/train_manifest.jsonl",
|
| 19 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0",
|
| 20 |
+
"max_records": null,
|
| 21 |
+
"teacher_model_name": "Wan2.1-T2V-1.3B",
|
| 22 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 23 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 24 |
+
"strict_teacher_load": true,
|
| 25 |
+
"source_block_ids": [
|
| 26 |
+
1,
|
| 27 |
+
28
|
| 28 |
+
],
|
| 29 |
+
"timestep_shift": 5.0,
|
| 30 |
+
"local_attn_size": -1,
|
| 31 |
+
"sink_size": 0,
|
| 32 |
+
"latent_height": 60,
|
| 33 |
+
"latent_width": 104,
|
| 34 |
+
"max_steps": 2000,
|
| 35 |
+
"batch_size": 1,
|
| 36 |
+
"gradient_accumulation_steps": 1,
|
| 37 |
+
"num_workers": 0,
|
| 38 |
+
"pin_memory": false,
|
| 39 |
+
"persistent_workers": false,
|
| 40 |
+
"fusion_lr": 0.0001,
|
| 41 |
+
"blocks_lr": 1e-05,
|
| 42 |
+
"beta1": 0.9,
|
| 43 |
+
"beta2": 0.95,
|
| 44 |
+
"weight_decay": 0.01,
|
| 45 |
+
"scheduler_warmup_steps": 100,
|
| 46 |
+
"block_warmup_steps": 100,
|
| 47 |
+
"hidden_loss_weight": 0.1,
|
| 48 |
+
"flow_loss_weight": 1.0,
|
| 49 |
+
"grad_clip": 1.0,
|
| 50 |
+
"fp32_trainable_params": true,
|
| 51 |
+
"gradient_checkpointing": false,
|
| 52 |
+
"find_unused_parameters": false,
|
| 53 |
+
"log_every": 10,
|
| 54 |
+
"save_every": 100,
|
| 55 |
+
"keep_snapshots": 20,
|
| 56 |
+
"resume": null,
|
| 57 |
+
"use_wandb": false,
|
| 58 |
+
"wandb_project": "Self-Forcing-Predictor-v4",
|
| 59 |
+
"wandb_entity": null,
|
| 60 |
+
"wandb_name": "predictor-v4-prefeature-blocks1-28-prompts1000-seed0",
|
| 61 |
+
"supervision_pairs": [
|
| 62 |
+
[
|
| 63 |
+
0,
|
| 64 |
+
1
|
| 65 |
+
],
|
| 66 |
+
[
|
| 67 |
+
1,
|
| 68 |
+
2
|
| 69 |
+
],
|
| 70 |
+
[
|
| 71 |
+
2,
|
| 72 |
+
3
|
| 73 |
+
]
|
| 74 |
+
],
|
| 75 |
+
"manifest_records": 6000,
|
| 76 |
+
"dataset_pairs": 18000,
|
| 77 |
+
"steps_per_epoch_per_rank": 2250,
|
| 78 |
+
"world_size": 8,
|
| 79 |
+
"global_batch_size": 8,
|
| 80 |
+
"trainable_parameters": 104684032,
|
| 81 |
+
"parameter_groups": {
|
| 82 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 83 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 84 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 85 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 86 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 87 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 88 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 89 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 90 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 91 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 92 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 93 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 94 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 95 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 96 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 97 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 98 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 99 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 100 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 101 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 102 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 103 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 104 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 105 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 106 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 107 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 108 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 109 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 110 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 111 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 112 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 113 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 114 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 115 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 116 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 117 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 118 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 119 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 120 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 121 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 122 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 123 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 124 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 125 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 126 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 127 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 128 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 129 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 130 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 131 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 132 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 133 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 134 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 135 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 136 |
+
"residual_out.weight": "fusion_residual",
|
| 137 |
+
"residual_out.bias": "fusion_residual"
|
| 138 |
+
},
|
| 139 |
+
"activation_dtype": "bfloat16",
|
| 140 |
+
"trainable_parameter_dtype": "float32"
|
| 141 |
+
}
|
| 142 |
+
{"avg_micro_time_s": 0.8126278872659896, "epoch": 0, "flow_mse": 1.7932306975126266, "global_step": 1, "grad_norm_max": 1.3163436651229858, "hidden_mse": 0.36144113540649414, "loss": 1.8293748050928116, "lr_blocks": 0.0, "lr_fusion": 2.0000000000000003e-06, "peak_memory_gib": 3.228342056274414}
|
| 143 |
+
{"avg_micro_time_s": 0.12664093374526902, "epoch": 0, "flow_mse": 1.7424163752131991, "global_step": 10, "grad_norm_max": 1.57919442653656, "hidden_mse": 0.36704767619570094, "loss": 1.779121145606041, "lr_blocks": 0.0, "lr_fusion": 1.1000000000000001e-05, "peak_memory_gib": 4.022468090057373}
|
| 144 |
+
{"avg_micro_time_s": 0.08021761359123047, "epoch": 0, "flow_mse": 1.6971245273947717, "global_step": 20, "grad_norm_max": 1.6758753061294556, "hidden_mse": 0.38903381638228896, "loss": 1.7360279053449632, "lr_blocks": 0.0, "lr_fusion": 2.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 145 |
+
{"avg_micro_time_s": 0.08832647445378826, "epoch": 0, "flow_mse": 1.6122777327895166, "global_step": 30, "grad_norm_max": 1.6740425825119019, "hidden_mse": 0.5868539184331895, "loss": 1.6709631234407425, "lr_blocks": 0.0, "lr_fusion": 3.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 146 |
+
{"avg_micro_time_s": 0.07797400094132173, "epoch": 0, "flow_mse": 1.4097208425402643, "global_step": 40, "grad_norm_max": 1.1768473386764526, "hidden_mse": 1.404243987798691, "loss": 1.5501452445983888, "lr_blocks": 0.0, "lr_fusion": 4.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 147 |
+
{"avg_micro_time_s": 0.09865606077801203, "epoch": 0, "flow_mse": 1.3578923508524896, "global_step": 50, "grad_norm_max": 1.7214945554733276, "hidden_mse": 1.65730447024107, "loss": 1.5236227929592134, "lr_blocks": 0.0, "lr_fusion": 5.1000000000000006e-05, "peak_memory_gib": 4.022468090057373}
|
| 148 |
+
{"avg_micro_time_s": 0.07141927356860833, "epoch": 0, "flow_mse": 1.3065032735466957, "global_step": 60, "grad_norm_max": 1.483780860900879, "hidden_mse": 1.6305436596274376, "loss": 1.4695576429367065, "lr_blocks": 0.0, "lr_fusion": 6.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 149 |
+
{"avg_micro_time_s": 0.086248747058562, "epoch": 0, "flow_mse": 1.259536074101925, "global_step": 70, "grad_norm_max": 2.2952280044555664, "hidden_mse": 1.538761314749718, "loss": 1.4134122028946878, "lr_blocks": 0.0, "lr_fusion": 7.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 150 |
+
{"avg_micro_time_s": 0.07471126251039095, "epoch": 0, "flow_mse": 1.1969433784484864, "global_step": 80, "grad_norm_max": 1.7487881183624268, "hidden_mse": 1.3721626818180086, "loss": 1.3341596513986589, "lr_blocks": 0.0, "lr_fusion": 8.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 151 |
+
{"avg_micro_time_s": 0.07639330443926157, "epoch": 0, "flow_mse": 1.132824356108904, "global_step": 90, "grad_norm_max": 2.0975935459136963, "hidden_mse": 1.2404362589120865, "loss": 1.256867977976799, "lr_blocks": 0.0, "lr_fusion": 9.1e-05, "peak_memory_gib": 4.022468090057373}
|
| 152 |
+
Terminated
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/launcher.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
*
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/debug/debug.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/config.yaml
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
activation_dtype:
|
| 2 |
+
desc: ''
|
| 3 |
+
sort: 47
|
| 4 |
+
value: bfloat16
|
| 5 |
+
beta1:
|
| 6 |
+
desc: ''
|
| 7 |
+
sort: 16
|
| 8 |
+
value: 0.9
|
| 9 |
+
beta2:
|
| 10 |
+
desc: ''
|
| 11 |
+
sort: 17
|
| 12 |
+
value: 0.95
|
| 13 |
+
blocks_lr:
|
| 14 |
+
desc: ''
|
| 15 |
+
sort: 15
|
| 16 |
+
value: 5.0e-06
|
| 17 |
+
cases_path:
|
| 18 |
+
desc: ''
|
| 19 |
+
sort: 2
|
| 20 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/cases.jsonl
|
| 21 |
+
data_root:
|
| 22 |
+
desc: ''
|
| 23 |
+
sort: 1
|
| 24 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 25 |
+
default_config:
|
| 26 |
+
desc: ''
|
| 27 |
+
sort: 5
|
| 28 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/default_config.yaml
|
| 29 |
+
dmd_config:
|
| 30 |
+
desc: ''
|
| 31 |
+
sort: 6
|
| 32 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/self_forcing_dmd.yaml
|
| 33 |
+
exact_online_timesteps:
|
| 34 |
+
desc: ''
|
| 35 |
+
sort: 40
|
| 36 |
+
value:
|
| 37 |
+
- 1000.0
|
| 38 |
+
- 937.5
|
| 39 |
+
- 833.3333129882812
|
| 40 |
+
- 625.0
|
| 41 |
+
flow_loss_weight:
|
| 42 |
+
desc: ''
|
| 43 |
+
sort: 21
|
| 44 |
+
value: 1.0
|
| 45 |
+
fusion_lr:
|
| 46 |
+
desc: ''
|
| 47 |
+
sort: 14
|
| 48 |
+
value: 5.0e-05
|
| 49 |
+
global_batch_trajectories:
|
| 50 |
+
desc: ''
|
| 51 |
+
sort: 43
|
| 52 |
+
value: 8
|
| 53 |
+
grad_clip:
|
| 54 |
+
desc: ''
|
| 55 |
+
sort: 22
|
| 56 |
+
value: 1.0
|
| 57 |
+
hidden_loss_weight:
|
| 58 |
+
desc: ''
|
| 59 |
+
sort: 20
|
| 60 |
+
value: 0.1
|
| 61 |
+
keep_snapshots:
|
| 62 |
+
desc: ''
|
| 63 |
+
sort: 28
|
| 64 |
+
value: 20
|
| 65 |
+
log_every:
|
| 66 |
+
desc: ''
|
| 67 |
+
sort: 26
|
| 68 |
+
value: 1
|
| 69 |
+
max_cases:
|
| 70 |
+
desc: ''
|
| 71 |
+
sort: 3
|
| 72 |
+
value: {}
|
| 73 |
+
max_steps:
|
| 74 |
+
desc: ''
|
| 75 |
+
sort: 13
|
| 76 |
+
value: 2000
|
| 77 |
+
num_workers:
|
| 78 |
+
desc: ''
|
| 79 |
+
sort: 23
|
| 80 |
+
value: 0
|
| 81 |
+
offline_target_timestep_semantics:
|
| 82 |
+
desc: ''
|
| 83 |
+
sort: 41
|
| 84 |
+
value: saved_stage1_int64
|
| 85 |
+
output_dir:
|
| 86 |
+
desc: ''
|
| 87 |
+
sort: 4
|
| 88 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_offline_ffff_fppp_1000_seed0
|
| 89 |
+
parameter_groups:
|
| 90 |
+
desc: ''
|
| 91 |
+
sort: 45
|
| 92 |
+
value:
|
| 93 |
+
feature_fusion.anchor_norm.bias: fusion_residual
|
| 94 |
+
feature_fusion.anchor_norm.weight: fusion_residual
|
| 95 |
+
feature_fusion.current_norm.bias: fusion_residual
|
| 96 |
+
feature_fusion.current_norm.weight: fusion_residual
|
| 97 |
+
feature_fusion.mlp.0.bias: fusion_residual
|
| 98 |
+
feature_fusion.mlp.0.weight: fusion_residual
|
| 99 |
+
feature_fusion.mlp.2.bias: fusion_residual
|
| 100 |
+
feature_fusion.mlp.2.weight: fusion_residual
|
| 101 |
+
feature_fusion.previous_norm.bias: fusion_residual
|
| 102 |
+
feature_fusion.previous_norm.weight: fusion_residual
|
| 103 |
+
predictor_blocks.0.cross_attn.norm_q.weight: blocks
|
| 104 |
+
predictor_blocks.0.cross_attn.o.bias: blocks
|
| 105 |
+
predictor_blocks.0.cross_attn.o.weight: blocks
|
| 106 |
+
predictor_blocks.0.cross_attn.q.bias: blocks
|
| 107 |
+
predictor_blocks.0.cross_attn.q.weight: blocks
|
| 108 |
+
predictor_blocks.0.ffn.0.bias: blocks
|
| 109 |
+
predictor_blocks.0.ffn.0.weight: blocks
|
| 110 |
+
predictor_blocks.0.ffn.2.bias: blocks
|
| 111 |
+
predictor_blocks.0.ffn.2.weight: blocks
|
| 112 |
+
predictor_blocks.0.modulation: blocks
|
| 113 |
+
predictor_blocks.0.norm3.bias: blocks
|
| 114 |
+
predictor_blocks.0.norm3.weight: blocks
|
| 115 |
+
predictor_blocks.0.self_attn.k.bias: blocks
|
| 116 |
+
predictor_blocks.0.self_attn.k.weight: blocks
|
| 117 |
+
predictor_blocks.0.self_attn.norm_k.weight: blocks
|
| 118 |
+
predictor_blocks.0.self_attn.norm_q.weight: blocks
|
| 119 |
+
predictor_blocks.0.self_attn.o.bias: blocks
|
| 120 |
+
predictor_blocks.0.self_attn.o.weight: blocks
|
| 121 |
+
predictor_blocks.0.self_attn.q.bias: blocks
|
| 122 |
+
predictor_blocks.0.self_attn.q.weight: blocks
|
| 123 |
+
predictor_blocks.0.self_attn.v.bias: blocks
|
| 124 |
+
predictor_blocks.0.self_attn.v.weight: blocks
|
| 125 |
+
predictor_blocks.1.cross_attn.norm_q.weight: blocks
|
| 126 |
+
predictor_blocks.1.cross_attn.o.bias: blocks
|
| 127 |
+
predictor_blocks.1.cross_attn.o.weight: blocks
|
| 128 |
+
predictor_blocks.1.cross_attn.q.bias: blocks
|
| 129 |
+
predictor_blocks.1.cross_attn.q.weight: blocks
|
| 130 |
+
predictor_blocks.1.ffn.0.bias: blocks
|
| 131 |
+
predictor_blocks.1.ffn.0.weight: blocks
|
| 132 |
+
predictor_blocks.1.ffn.2.bias: blocks
|
| 133 |
+
predictor_blocks.1.ffn.2.weight: blocks
|
| 134 |
+
predictor_blocks.1.modulation: blocks
|
| 135 |
+
predictor_blocks.1.norm3.bias: blocks
|
| 136 |
+
predictor_blocks.1.norm3.weight: blocks
|
| 137 |
+
predictor_blocks.1.self_attn.k.bias: blocks
|
| 138 |
+
predictor_blocks.1.self_attn.k.weight: blocks
|
| 139 |
+
predictor_blocks.1.self_attn.norm_k.weight: blocks
|
| 140 |
+
predictor_blocks.1.self_attn.norm_q.weight: blocks
|
| 141 |
+
predictor_blocks.1.self_attn.o.bias: blocks
|
| 142 |
+
predictor_blocks.1.self_attn.o.weight: blocks
|
| 143 |
+
predictor_blocks.1.self_attn.q.bias: blocks
|
| 144 |
+
predictor_blocks.1.self_attn.q.weight: blocks
|
| 145 |
+
predictor_blocks.1.self_attn.v.bias: blocks
|
| 146 |
+
predictor_blocks.1.self_attn.v.weight: blocks
|
| 147 |
+
residual_out.bias: fusion_residual
|
| 148 |
+
residual_out.weight: fusion_residual
|
| 149 |
+
persistent_cache_semantics:
|
| 150 |
+
desc: ''
|
| 151 |
+
sort: 39
|
| 152 |
+
value: final_clean_full_pass_only
|
| 153 |
+
persistent_workers:
|
| 154 |
+
desc: ''
|
| 155 |
+
sort: 25
|
| 156 |
+
value: false
|
| 157 |
+
pin_memory:
|
| 158 |
+
desc: ''
|
| 159 |
+
sort: 24
|
| 160 |
+
value: false
|
| 161 |
+
resume:
|
| 162 |
+
desc: ''
|
| 163 |
+
sort: 29
|
| 164 |
+
value: {}
|
| 165 |
+
save_every:
|
| 166 |
+
desc: ''
|
| 167 |
+
sort: 27
|
| 168 |
+
value: 100
|
| 169 |
+
source_block_ids:
|
| 170 |
+
desc: ''
|
| 171 |
+
sort: 10
|
| 172 |
+
value:
|
| 173 |
+
- 1
|
| 174 |
+
- 28
|
| 175 |
+
stage1_expected_step:
|
| 176 |
+
desc: ''
|
| 177 |
+
sort: 12
|
| 178 |
+
value: 2000
|
| 179 |
+
stage1_training_state:
|
| 180 |
+
desc: ''
|
| 181 |
+
sort: 11
|
| 182 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/training_latest.pt
|
| 183 |
+
strict_teacher_load:
|
| 184 |
+
desc: ''
|
| 185 |
+
sort: 9
|
| 186 |
+
value: true
|
| 187 |
+
supervised_states_per_optimizer_step:
|
| 188 |
+
desc: ''
|
| 189 |
+
sort: 44
|
| 190 |
+
value: 144
|
| 191 |
+
supervision_mode:
|
| 192 |
+
desc: ''
|
| 193 |
+
sort: 0
|
| 194 |
+
value: offline_ffff
|
| 195 |
+
swanlab_description:
|
| 196 |
+
desc: ''
|
| 197 |
+
sort: 34
|
| 198 |
+
value: Self-Forcing Predictor-v4 Stage 2 FPPP rollout with fixed offline FFFF hidden
|
| 199 |
+
and flow targets
|
| 200 |
+
swanlab_experiment:
|
| 201 |
+
desc: ''
|
| 202 |
+
sort: 33
|
| 203 |
+
value: predictor-v4-stage2-offline-ffff-targets-detached-chunk-timestep-fppp-prompts1000-seed0
|
| 204 |
+
swanlab_mode:
|
| 205 |
+
desc: ''
|
| 206 |
+
sort: 36
|
| 207 |
+
value: cloud
|
| 208 |
+
swanlab_project:
|
| 209 |
+
desc: ''
|
| 210 |
+
sort: 31
|
| 211 |
+
value: Self-Forcing-Predictor-v4
|
| 212 |
+
swanlab_tags:
|
| 213 |
+
desc: ''
|
| 214 |
+
sort: 35
|
| 215 |
+
value:
|
| 216 |
+
- predictor-v4
|
| 217 |
+
- stage2
|
| 218 |
+
- fppp-rollout
|
| 219 |
+
- detached-timestep
|
| 220 |
+
- detached-chunk
|
| 221 |
+
- offline-ffff-targets
|
| 222 |
+
- blocks-1-28
|
| 223 |
+
- 8gpu
|
| 224 |
+
- seed0
|
| 225 |
+
swanlab_workspace:
|
| 226 |
+
desc: ''
|
| 227 |
+
sort: 32
|
| 228 |
+
value: {}
|
| 229 |
+
teacher_checkpoint:
|
| 230 |
+
desc: ''
|
| 231 |
+
sort: 7
|
| 232 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt
|
| 233 |
+
teacher_checkpoint_key:
|
| 234 |
+
desc: ''
|
| 235 |
+
sort: 8
|
| 236 |
+
value: generator_ema
|
| 237 |
+
train_predictor_steps:
|
| 238 |
+
desc: ''
|
| 239 |
+
sort: 38
|
| 240 |
+
value:
|
| 241 |
+
- 1
|
| 242 |
+
- 2
|
| 243 |
+
- 3
|
| 244 |
+
trainable_parameter_dtype:
|
| 245 |
+
desc: ''
|
| 246 |
+
sort: 48
|
| 247 |
+
value: float32
|
| 248 |
+
trainable_parameters:
|
| 249 |
+
desc: ''
|
| 250 |
+
sort: 46
|
| 251 |
+
value: 104684032
|
| 252 |
+
trainer_semantics:
|
| 253 |
+
desc: ''
|
| 254 |
+
sort: 37
|
| 255 |
+
value: chunk0_ffff_chunks1-6_fppp_detached_timestep_and_chunk
|
| 256 |
+
use_swanlab:
|
| 257 |
+
desc: ''
|
| 258 |
+
sort: 30
|
| 259 |
+
value: true
|
| 260 |
+
warmup_steps:
|
| 261 |
+
desc: ''
|
| 262 |
+
sort: 19
|
| 263 |
+
value: 100
|
| 264 |
+
weight_decay:
|
| 265 |
+
desc: ''
|
| 266 |
+
sort: 18
|
| 267 |
+
value: 0.01
|
| 268 |
+
world_size:
|
| 269 |
+
desc: ''
|
| 270 |
+
sort: 42
|
| 271 |
+
value: 8
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/requirements.txt
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
accelerate==1.14.0
|
| 2 |
+
aiohappyeyeballs==2.7.1
|
| 3 |
+
aiohttp==3.14.1
|
| 4 |
+
aiosignal==1.4.0
|
| 5 |
+
annotated-doc==0.0.4
|
| 6 |
+
annotated-types==0.7.0
|
| 7 |
+
antlr4-python3-runtime==4.9.3
|
| 8 |
+
anyio==4.14.1
|
| 9 |
+
async-timeout==5.0.1
|
| 10 |
+
attrs==26.1.0
|
| 11 |
+
av==13.1.0
|
| 12 |
+
bidict==0.23.1
|
| 13 |
+
blinker==1.9.0
|
| 14 |
+
certifi==2026.6.17
|
| 15 |
+
cffi==2.0.0
|
| 16 |
+
charset-normalizer==3.4.7
|
| 17 |
+
click==8.4.2
|
| 18 |
+
clip==1.0
|
| 19 |
+
coloredlogs==15.0.1
|
| 20 |
+
contourpy==1.3.2
|
| 21 |
+
cryptography==49.0.0
|
| 22 |
+
cuda-bindings==13.3.1
|
| 23 |
+
cuda-pathfinder==1.5.6
|
| 24 |
+
cuda-toolkit==13.0.2
|
| 25 |
+
cycler==0.12.1
|
| 26 |
+
dashscope==1.26.2
|
| 27 |
+
diffusers==0.31.0
|
| 28 |
+
dominate==2.9.1
|
| 29 |
+
easydict==1.13
|
| 30 |
+
einops==0.8.2
|
| 31 |
+
exceptiongroup==1.3.1
|
| 32 |
+
filelock==3.29.0
|
| 33 |
+
flash_attn==2.8.3.post1
|
| 34 |
+
Flask==3.1.3
|
| 35 |
+
Flask-SocketIO==5.6.1
|
| 36 |
+
flatbuffers==25.12.19
|
| 37 |
+
fonttools==4.63.0
|
| 38 |
+
frozenlist==1.8.0
|
| 39 |
+
fsspec==2026.4.0
|
| 40 |
+
ftfy==6.3.1
|
| 41 |
+
gitdb==4.0.12
|
| 42 |
+
GitPython==3.1.50
|
| 43 |
+
h11==0.16.0
|
| 44 |
+
hf-xet==1.5.1
|
| 45 |
+
httpcore==1.0.9
|
| 46 |
+
httpx==0.28.1
|
| 47 |
+
httpx-sse==0.4.3
|
| 48 |
+
huggingface_hub==1.21.0
|
| 49 |
+
humanfriendly==10.0
|
| 50 |
+
idna==3.18
|
| 51 |
+
ImageIO==2.37.3
|
| 52 |
+
imageio-ffmpeg==0.6.0
|
| 53 |
+
importlib_metadata==9.0.0
|
| 54 |
+
itsdangerous==2.2.0
|
| 55 |
+
Jinja2==3.1.6
|
| 56 |
+
kiwisolver==1.5.0
|
| 57 |
+
lazy-loader==0.5
|
| 58 |
+
lmdb==2.2.1
|
| 59 |
+
Mako==1.3.12
|
| 60 |
+
markdown-it-py==4.2.0
|
| 61 |
+
MarkupSafe==3.0.3
|
| 62 |
+
matplotlib==3.10.9
|
| 63 |
+
mdurl==0.1.2
|
| 64 |
+
ml_dtypes==0.5.4
|
| 65 |
+
mpmath==1.3.0
|
| 66 |
+
multidict==6.7.1
|
| 67 |
+
networkx==3.4.2
|
| 68 |
+
ninja==1.13.0
|
| 69 |
+
numpy==1.24.4
|
| 70 |
+
nvidia-cublas==13.1.1.3
|
| 71 |
+
nvidia-cublas-cu12==12.1.3.1
|
| 72 |
+
nvidia-cuda-cupti==13.0.85
|
| 73 |
+
nvidia-cuda-cupti-cu12==12.1.105
|
| 74 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 75 |
+
nvidia-cuda-nvrtc-cu12==12.1.105
|
| 76 |
+
nvidia-cuda-runtime==13.0.96
|
| 77 |
+
nvidia-cuda-runtime-cu12==12.1.105
|
| 78 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 79 |
+
nvidia-cudnn-cu13==9.20.0.48
|
| 80 |
+
nvidia-cufft==12.0.0.61
|
| 81 |
+
nvidia-cufft-cu12==11.0.2.54
|
| 82 |
+
nvidia-cufile==1.15.1.6
|
| 83 |
+
nvidia-curand==10.4.0.35
|
| 84 |
+
nvidia-curand-cu12==10.3.2.106
|
| 85 |
+
nvidia-cusolver==12.0.4.66
|
| 86 |
+
nvidia-cusolver-cu12==11.4.5.107
|
| 87 |
+
nvidia-cusparse==12.6.3.3
|
| 88 |
+
nvidia-cusparse-cu12==12.1.0.106
|
| 89 |
+
nvidia-cusparselt-cu13==0.8.1
|
| 90 |
+
nvidia-ml-py==13.610.43
|
| 91 |
+
nvidia-nccl-cu12==2.21.5
|
| 92 |
+
nvidia-nccl-cu13==2.29.7
|
| 93 |
+
nvidia-nvjitlink==13.0.88
|
| 94 |
+
nvidia-nvjitlink-cu12==12.9.86
|
| 95 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 96 |
+
nvidia-nvtx==13.0.85
|
| 97 |
+
nvidia-nvtx-cu12==12.1.105
|
| 98 |
+
nvidia-tensorrt==99.0.0
|
| 99 |
+
omegaconf==2.3.1
|
| 100 |
+
onnx==1.22.0
|
| 101 |
+
onnx-ir==0.2.1
|
| 102 |
+
onnxconverter-common==1.16.0
|
| 103 |
+
onnxruntime==1.23.2
|
| 104 |
+
onnxscript==0.7.1
|
| 105 |
+
open_clip_torch==3.3.0
|
| 106 |
+
opencv-python==4.11.0.86
|
| 107 |
+
orjson==3.11.9
|
| 108 |
+
packaging==26.2
|
| 109 |
+
pillow==12.2.0
|
| 110 |
+
pip==26.1.2
|
| 111 |
+
platformdirs==4.10.0
|
| 112 |
+
prettytable==3.18.0
|
| 113 |
+
propcache==0.5.2
|
| 114 |
+
protobuf==6.33.6
|
| 115 |
+
psutil==7.2.2
|
| 116 |
+
pwinput==1.0.3
|
| 117 |
+
pycocotools==2.0.11
|
| 118 |
+
pycparser==3.0
|
| 119 |
+
pycuda==2026.1
|
| 120 |
+
pydantic==2.10.6
|
| 121 |
+
pydantic_core==2.27.2
|
| 122 |
+
pydantic-settings==2.14.2
|
| 123 |
+
pyecharts==2.1.0
|
| 124 |
+
Pygments==2.20.0
|
| 125 |
+
pyparsing==3.3.2
|
| 126 |
+
python-dateutil==2.9.0.post0
|
| 127 |
+
python-dotenv==1.2.2
|
| 128 |
+
python-engineio==4.13.3
|
| 129 |
+
python-socketio==5.16.3
|
| 130 |
+
pytools==2026.1.1
|
| 131 |
+
PyYAML==6.0.3
|
| 132 |
+
regex==2026.6.28
|
| 133 |
+
requests==2.34.2
|
| 134 |
+
rich==15.0.0
|
| 135 |
+
safetensors==0.8.0
|
| 136 |
+
scikit-image==0.25.2
|
| 137 |
+
scipy==1.15.3
|
| 138 |
+
self_forcing==0.0.1
|
| 139 |
+
sentencepiece==0.2.1
|
| 140 |
+
sentry-sdk==2.64.0
|
| 141 |
+
setuptools==81.0.0
|
| 142 |
+
shellingham==1.5.4
|
| 143 |
+
simple-websocket==1.1.0
|
| 144 |
+
simplejson==4.1.1
|
| 145 |
+
siphash24==1.8
|
| 146 |
+
six==1.17.0
|
| 147 |
+
smmap==5.0.3
|
| 148 |
+
starlette==1.3.1
|
| 149 |
+
swanlab==0.9.0
|
| 150 |
+
sympy==1.13.1
|
| 151 |
+
tensorrt==11.1.0.106
|
| 152 |
+
tensorrt_cu13==11.1.0.106
|
| 153 |
+
tensorrt_cu13_bindings==11.1.0.106
|
| 154 |
+
tensorrt_cu13_libs==11.1.0.106
|
| 155 |
+
tifffile==2025.5.10
|
| 156 |
+
timm==1.0.27
|
| 157 |
+
tokenizers==0.22.2
|
| 158 |
+
torch==2.5.1+cu121
|
| 159 |
+
torchao==0.17.0
|
| 160 |
+
torchvision==0.20.1+cu121
|
| 161 |
+
tqdm==4.68.3
|
| 162 |
+
transformers==5.12.1
|
| 163 |
+
triton==3.1.0
|
| 164 |
+
typer==0.25.1
|
| 165 |
+
typing_extensions==4.15.0
|
| 166 |
+
typing-inspection==0.4.2
|
| 167 |
+
urllib3==2.7.0
|
| 168 |
+
wandb==0.28.0
|
| 169 |
+
watchdog==6.0.0
|
| 170 |
+
wcwidth==0.8.2
|
| 171 |
+
websocket-client==1.9.0
|
| 172 |
+
Werkzeug==3.1.8
|
| 173 |
+
wheel==0.47.0
|
| 174 |
+
wrapt==2.2.2
|
| 175 |
+
wsproto==1.3.2
|
| 176 |
+
yarl==1.24.2
|
| 177 |
+
zipp==4.1.0
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au/files/swanlab-metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_version":2,"hardware":{"apple_silicon":null,"cpu":{"brand":"AMD EPYC 7R13 Processor","physical_count":96,"logical_count":192},"memory":{"total":1999,"total_unit":"GB"},"accelerators":[{"vendor":"nvidia","devices":[{"index":0,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":1,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":2,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":3,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":4,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":5,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":6,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":7,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"}],"version":"580.167.08","cuda_version":"13.0","cann_version":null}]},"runtime":{"os":"Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.34","os_pretty":"Amazon Linux 2023.12.20260611","hostname":"ip-172-31-49-199.us-west-2.compute.internal","pid":3712319,"cwd":"/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing","python_version":"3.10.20","python_verbose":"3.10.20 | packaged by conda-forge | (main, Jun 11 2026, 03:31:56) [GCC 14.3.0]","python_executable":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python","command":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python -u train.py --config_path /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/predictor_v4_stage2_offline_ffff_fppp.yaml --disable-wandb --config_override predictor_v4_rollout.output_dir=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_offline_ffff_fppp_1000_seed0 --config_override predictor_v4_rollout.max_steps=2000 --config_override predictor_v4_rollout.max_cases=null --config_override predictor_v4_rollout.use_swanlab=true"},"git":{"remote_url":"https://github.com/guandeh17/Self-Forcing","branch":"main","commit":"33593df3e81fa3ec10239271dd2c100facac6de1"},"swanlab":{"version":"0.9.0","run_dir":"/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_offline_ffff_fppp_1000_seed0/swanlab/run-20260726_100449-lhs9t5au"}}
|
checkpoints/predictor_v4_stage2_offline_ffff_fppp_1000_seed0/train_config.json
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_dtype": "bfloat16",
|
| 3 |
+
"beta1": 0.9,
|
| 4 |
+
"beta2": 0.95,
|
| 5 |
+
"blocks_lr": 5e-06,
|
| 6 |
+
"cases_path": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/cases.jsonl",
|
| 7 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 8 |
+
"default_config": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/default_config.yaml",
|
| 9 |
+
"dmd_config": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/self_forcing_dmd.yaml",
|
| 10 |
+
"exact_online_timesteps": [
|
| 11 |
+
1000.0,
|
| 12 |
+
937.5,
|
| 13 |
+
833.3333129882812,
|
| 14 |
+
625.0
|
| 15 |
+
],
|
| 16 |
+
"flow_loss_weight": 1.0,
|
| 17 |
+
"fusion_lr": 5e-05,
|
| 18 |
+
"global_batch_trajectories": 8,
|
| 19 |
+
"grad_clip": 1.0,
|
| 20 |
+
"hidden_loss_weight": 0.1,
|
| 21 |
+
"keep_snapshots": 20,
|
| 22 |
+
"log_every": 1,
|
| 23 |
+
"max_cases": null,
|
| 24 |
+
"max_steps": 2000,
|
| 25 |
+
"num_workers": 0,
|
| 26 |
+
"offline_target_timestep_semantics": "saved_stage1_int64",
|
| 27 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_offline_ffff_fppp_1000_seed0",
|
| 28 |
+
"parameter_groups": {
|
| 29 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 30 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 31 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 32 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 33 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 34 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 35 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 36 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 37 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 38 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 39 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 40 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 41 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 42 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 43 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 44 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 45 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 46 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 47 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 48 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 49 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 50 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 51 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 52 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 53 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 54 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 55 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 56 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 57 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 58 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 59 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 60 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 61 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 62 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 63 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 64 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 65 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 66 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 67 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 68 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 69 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 70 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 71 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 72 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 73 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 74 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 75 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 76 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 77 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 78 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 79 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 80 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 81 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 82 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 83 |
+
"residual_out.bias": "fusion_residual",
|
| 84 |
+
"residual_out.weight": "fusion_residual"
|
| 85 |
+
},
|
| 86 |
+
"persistent_cache_semantics": "final_clean_full_pass_only",
|
| 87 |
+
"persistent_workers": false,
|
| 88 |
+
"pin_memory": false,
|
| 89 |
+
"resume": null,
|
| 90 |
+
"save_every": 100,
|
| 91 |
+
"source_block_ids": [
|
| 92 |
+
1,
|
| 93 |
+
28
|
| 94 |
+
],
|
| 95 |
+
"stage1_expected_step": 2000,
|
| 96 |
+
"stage1_training_state": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/training_latest.pt",
|
| 97 |
+
"strict_teacher_load": true,
|
| 98 |
+
"supervised_states_per_optimizer_step": 144,
|
| 99 |
+
"supervision_mode": "offline_ffff",
|
| 100 |
+
"swanlab_description": "Self-Forcing Predictor-v4 Stage 2 FPPP rollout with fixed offline FFFF hidden and flow targets",
|
| 101 |
+
"swanlab_experiment": "predictor-v4-stage2-offline-ffff-targets-detached-chunk-timestep-fppp-prompts1000-seed0",
|
| 102 |
+
"swanlab_mode": "cloud",
|
| 103 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 104 |
+
"swanlab_run_id": "lhs9t5au",
|
| 105 |
+
"swanlab_tags": [
|
| 106 |
+
"predictor-v4",
|
| 107 |
+
"stage2",
|
| 108 |
+
"fppp-rollout",
|
| 109 |
+
"detached-timestep",
|
| 110 |
+
"detached-chunk",
|
| 111 |
+
"offline-ffff-targets",
|
| 112 |
+
"blocks-1-28",
|
| 113 |
+
"8gpu",
|
| 114 |
+
"seed0"
|
| 115 |
+
],
|
| 116 |
+
"swanlab_workspace": null,
|
| 117 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 118 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 119 |
+
"train_predictor_steps": [
|
| 120 |
+
1,
|
| 121 |
+
2,
|
| 122 |
+
3
|
| 123 |
+
],
|
| 124 |
+
"trainable_parameter_dtype": "float32",
|
| 125 |
+
"trainable_parameters": 104684032,
|
| 126 |
+
"trainer_semantics": "chunk0_ffff_chunks1-6_fppp_detached_timestep_and_chunk",
|
| 127 |
+
"use_swanlab": true,
|
| 128 |
+
"warmup_steps": 100,
|
| 129 |
+
"weight_decay": 0.01,
|
| 130 |
+
"world_size": 8
|
| 131 |
+
}
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/launcher.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
*
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/debug/debug.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/config.yaml
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
activation_dtype:
|
| 2 |
+
desc: ''
|
| 3 |
+
sort: 47
|
| 4 |
+
value: bfloat16
|
| 5 |
+
beta1:
|
| 6 |
+
desc: ''
|
| 7 |
+
sort: 16
|
| 8 |
+
value: 0.9
|
| 9 |
+
beta2:
|
| 10 |
+
desc: ''
|
| 11 |
+
sort: 17
|
| 12 |
+
value: 0.95
|
| 13 |
+
blocks_lr:
|
| 14 |
+
desc: ''
|
| 15 |
+
sort: 15
|
| 16 |
+
value: 5.0e-06
|
| 17 |
+
cases_path:
|
| 18 |
+
desc: ''
|
| 19 |
+
sort: 2
|
| 20 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/cases.jsonl
|
| 21 |
+
data_root:
|
| 22 |
+
desc: ''
|
| 23 |
+
sort: 1
|
| 24 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0
|
| 25 |
+
default_config:
|
| 26 |
+
desc: ''
|
| 27 |
+
sort: 5
|
| 28 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/default_config.yaml
|
| 29 |
+
dmd_config:
|
| 30 |
+
desc: ''
|
| 31 |
+
sort: 6
|
| 32 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/self_forcing_dmd.yaml
|
| 33 |
+
exact_online_timesteps:
|
| 34 |
+
desc: ''
|
| 35 |
+
sort: 40
|
| 36 |
+
value:
|
| 37 |
+
- 1000.0
|
| 38 |
+
- 937.5
|
| 39 |
+
- 833.3333129882812
|
| 40 |
+
- 625.0
|
| 41 |
+
flow_loss_weight:
|
| 42 |
+
desc: ''
|
| 43 |
+
sort: 21
|
| 44 |
+
value: 1.0
|
| 45 |
+
fusion_lr:
|
| 46 |
+
desc: ''
|
| 47 |
+
sort: 14
|
| 48 |
+
value: 5.0e-05
|
| 49 |
+
global_batch_trajectories:
|
| 50 |
+
desc: ''
|
| 51 |
+
sort: 43
|
| 52 |
+
value: 8
|
| 53 |
+
grad_clip:
|
| 54 |
+
desc: ''
|
| 55 |
+
sort: 22
|
| 56 |
+
value: 1.0
|
| 57 |
+
hidden_loss_weight:
|
| 58 |
+
desc: ''
|
| 59 |
+
sort: 20
|
| 60 |
+
value: 0.1
|
| 61 |
+
keep_snapshots:
|
| 62 |
+
desc: ''
|
| 63 |
+
sort: 28
|
| 64 |
+
value: 20
|
| 65 |
+
log_every:
|
| 66 |
+
desc: ''
|
| 67 |
+
sort: 26
|
| 68 |
+
value: 1
|
| 69 |
+
max_cases:
|
| 70 |
+
desc: ''
|
| 71 |
+
sort: 3
|
| 72 |
+
value: {}
|
| 73 |
+
max_steps:
|
| 74 |
+
desc: ''
|
| 75 |
+
sort: 13
|
| 76 |
+
value: 2000
|
| 77 |
+
num_workers:
|
| 78 |
+
desc: ''
|
| 79 |
+
sort: 23
|
| 80 |
+
value: 0
|
| 81 |
+
offline_target_timestep_semantics:
|
| 82 |
+
desc: ''
|
| 83 |
+
sort: 41
|
| 84 |
+
value: {}
|
| 85 |
+
output_dir:
|
| 86 |
+
desc: ''
|
| 87 |
+
sort: 4
|
| 88 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_onpolicy_fppp_1000_seed0
|
| 89 |
+
parameter_groups:
|
| 90 |
+
desc: ''
|
| 91 |
+
sort: 45
|
| 92 |
+
value:
|
| 93 |
+
feature_fusion.anchor_norm.bias: fusion_residual
|
| 94 |
+
feature_fusion.anchor_norm.weight: fusion_residual
|
| 95 |
+
feature_fusion.current_norm.bias: fusion_residual
|
| 96 |
+
feature_fusion.current_norm.weight: fusion_residual
|
| 97 |
+
feature_fusion.mlp.0.bias: fusion_residual
|
| 98 |
+
feature_fusion.mlp.0.weight: fusion_residual
|
| 99 |
+
feature_fusion.mlp.2.bias: fusion_residual
|
| 100 |
+
feature_fusion.mlp.2.weight: fusion_residual
|
| 101 |
+
feature_fusion.previous_norm.bias: fusion_residual
|
| 102 |
+
feature_fusion.previous_norm.weight: fusion_residual
|
| 103 |
+
predictor_blocks.0.cross_attn.norm_q.weight: blocks
|
| 104 |
+
predictor_blocks.0.cross_attn.o.bias: blocks
|
| 105 |
+
predictor_blocks.0.cross_attn.o.weight: blocks
|
| 106 |
+
predictor_blocks.0.cross_attn.q.bias: blocks
|
| 107 |
+
predictor_blocks.0.cross_attn.q.weight: blocks
|
| 108 |
+
predictor_blocks.0.ffn.0.bias: blocks
|
| 109 |
+
predictor_blocks.0.ffn.0.weight: blocks
|
| 110 |
+
predictor_blocks.0.ffn.2.bias: blocks
|
| 111 |
+
predictor_blocks.0.ffn.2.weight: blocks
|
| 112 |
+
predictor_blocks.0.modulation: blocks
|
| 113 |
+
predictor_blocks.0.norm3.bias: blocks
|
| 114 |
+
predictor_blocks.0.norm3.weight: blocks
|
| 115 |
+
predictor_blocks.0.self_attn.k.bias: blocks
|
| 116 |
+
predictor_blocks.0.self_attn.k.weight: blocks
|
| 117 |
+
predictor_blocks.0.self_attn.norm_k.weight: blocks
|
| 118 |
+
predictor_blocks.0.self_attn.norm_q.weight: blocks
|
| 119 |
+
predictor_blocks.0.self_attn.o.bias: blocks
|
| 120 |
+
predictor_blocks.0.self_attn.o.weight: blocks
|
| 121 |
+
predictor_blocks.0.self_attn.q.bias: blocks
|
| 122 |
+
predictor_blocks.0.self_attn.q.weight: blocks
|
| 123 |
+
predictor_blocks.0.self_attn.v.bias: blocks
|
| 124 |
+
predictor_blocks.0.self_attn.v.weight: blocks
|
| 125 |
+
predictor_blocks.1.cross_attn.norm_q.weight: blocks
|
| 126 |
+
predictor_blocks.1.cross_attn.o.bias: blocks
|
| 127 |
+
predictor_blocks.1.cross_attn.o.weight: blocks
|
| 128 |
+
predictor_blocks.1.cross_attn.q.bias: blocks
|
| 129 |
+
predictor_blocks.1.cross_attn.q.weight: blocks
|
| 130 |
+
predictor_blocks.1.ffn.0.bias: blocks
|
| 131 |
+
predictor_blocks.1.ffn.0.weight: blocks
|
| 132 |
+
predictor_blocks.1.ffn.2.bias: blocks
|
| 133 |
+
predictor_blocks.1.ffn.2.weight: blocks
|
| 134 |
+
predictor_blocks.1.modulation: blocks
|
| 135 |
+
predictor_blocks.1.norm3.bias: blocks
|
| 136 |
+
predictor_blocks.1.norm3.weight: blocks
|
| 137 |
+
predictor_blocks.1.self_attn.k.bias: blocks
|
| 138 |
+
predictor_blocks.1.self_attn.k.weight: blocks
|
| 139 |
+
predictor_blocks.1.self_attn.norm_k.weight: blocks
|
| 140 |
+
predictor_blocks.1.self_attn.norm_q.weight: blocks
|
| 141 |
+
predictor_blocks.1.self_attn.o.bias: blocks
|
| 142 |
+
predictor_blocks.1.self_attn.o.weight: blocks
|
| 143 |
+
predictor_blocks.1.self_attn.q.bias: blocks
|
| 144 |
+
predictor_blocks.1.self_attn.q.weight: blocks
|
| 145 |
+
predictor_blocks.1.self_attn.v.bias: blocks
|
| 146 |
+
predictor_blocks.1.self_attn.v.weight: blocks
|
| 147 |
+
residual_out.bias: fusion_residual
|
| 148 |
+
residual_out.weight: fusion_residual
|
| 149 |
+
persistent_cache_semantics:
|
| 150 |
+
desc: ''
|
| 151 |
+
sort: 39
|
| 152 |
+
value: final_clean_full_pass_only
|
| 153 |
+
persistent_workers:
|
| 154 |
+
desc: ''
|
| 155 |
+
sort: 25
|
| 156 |
+
value: false
|
| 157 |
+
pin_memory:
|
| 158 |
+
desc: ''
|
| 159 |
+
sort: 24
|
| 160 |
+
value: false
|
| 161 |
+
resume:
|
| 162 |
+
desc: ''
|
| 163 |
+
sort: 29
|
| 164 |
+
value: {}
|
| 165 |
+
save_every:
|
| 166 |
+
desc: ''
|
| 167 |
+
sort: 27
|
| 168 |
+
value: 100
|
| 169 |
+
source_block_ids:
|
| 170 |
+
desc: ''
|
| 171 |
+
sort: 10
|
| 172 |
+
value:
|
| 173 |
+
- 1
|
| 174 |
+
- 28
|
| 175 |
+
stage1_expected_step:
|
| 176 |
+
desc: ''
|
| 177 |
+
sort: 12
|
| 178 |
+
value: 2000
|
| 179 |
+
stage1_training_state:
|
| 180 |
+
desc: ''
|
| 181 |
+
sort: 11
|
| 182 |
+
value: /mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/training_latest.pt
|
| 183 |
+
strict_teacher_load:
|
| 184 |
+
desc: ''
|
| 185 |
+
sort: 9
|
| 186 |
+
value: true
|
| 187 |
+
supervised_states_per_optimizer_step:
|
| 188 |
+
desc: ''
|
| 189 |
+
sort: 44
|
| 190 |
+
value: 144
|
| 191 |
+
supervision_mode:
|
| 192 |
+
desc: ''
|
| 193 |
+
sort: 0
|
| 194 |
+
value: onpolicy
|
| 195 |
+
swanlab_description:
|
| 196 |
+
desc: ''
|
| 197 |
+
sort: 34
|
| 198 |
+
value: Self-Forcing Predictor-v4 Stage 2 FPPP rollout with same-state online Full-Teacher
|
| 199 |
+
targets
|
| 200 |
+
swanlab_experiment:
|
| 201 |
+
desc: ''
|
| 202 |
+
sort: 33
|
| 203 |
+
value: predictor-v4-stage2-onpolicy-detached-chunk-timestep-fppp-prompts1000-seed0
|
| 204 |
+
swanlab_mode:
|
| 205 |
+
desc: ''
|
| 206 |
+
sort: 36
|
| 207 |
+
value: cloud
|
| 208 |
+
swanlab_project:
|
| 209 |
+
desc: ''
|
| 210 |
+
sort: 31
|
| 211 |
+
value: Self-Forcing-Predictor-v4
|
| 212 |
+
swanlab_tags:
|
| 213 |
+
desc: ''
|
| 214 |
+
sort: 35
|
| 215 |
+
value:
|
| 216 |
+
- predictor-v4
|
| 217 |
+
- stage2
|
| 218 |
+
- fppp-rollout
|
| 219 |
+
- detached-timestep
|
| 220 |
+
- detached-chunk
|
| 221 |
+
- onpolicy
|
| 222 |
+
- blocks-1-28
|
| 223 |
+
- 8gpu
|
| 224 |
+
- seed0
|
| 225 |
+
swanlab_workspace:
|
| 226 |
+
desc: ''
|
| 227 |
+
sort: 32
|
| 228 |
+
value: {}
|
| 229 |
+
teacher_checkpoint:
|
| 230 |
+
desc: ''
|
| 231 |
+
sort: 7
|
| 232 |
+
value: /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt
|
| 233 |
+
teacher_checkpoint_key:
|
| 234 |
+
desc: ''
|
| 235 |
+
sort: 8
|
| 236 |
+
value: generator_ema
|
| 237 |
+
train_predictor_steps:
|
| 238 |
+
desc: ''
|
| 239 |
+
sort: 38
|
| 240 |
+
value:
|
| 241 |
+
- 1
|
| 242 |
+
- 2
|
| 243 |
+
- 3
|
| 244 |
+
trainable_parameter_dtype:
|
| 245 |
+
desc: ''
|
| 246 |
+
sort: 48
|
| 247 |
+
value: float32
|
| 248 |
+
trainable_parameters:
|
| 249 |
+
desc: ''
|
| 250 |
+
sort: 46
|
| 251 |
+
value: 104684032
|
| 252 |
+
trainer_semantics:
|
| 253 |
+
desc: ''
|
| 254 |
+
sort: 37
|
| 255 |
+
value: chunk0_ffff_chunks1-6_fppp_detached_timestep_and_chunk
|
| 256 |
+
use_swanlab:
|
| 257 |
+
desc: ''
|
| 258 |
+
sort: 30
|
| 259 |
+
value: true
|
| 260 |
+
warmup_steps:
|
| 261 |
+
desc: ''
|
| 262 |
+
sort: 19
|
| 263 |
+
value: 100
|
| 264 |
+
weight_decay:
|
| 265 |
+
desc: ''
|
| 266 |
+
sort: 18
|
| 267 |
+
value: 0.01
|
| 268 |
+
world_size:
|
| 269 |
+
desc: ''
|
| 270 |
+
sort: 42
|
| 271 |
+
value: 8
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/requirements.txt
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
accelerate==1.14.0
|
| 2 |
+
aiohappyeyeballs==2.7.1
|
| 3 |
+
aiohttp==3.14.1
|
| 4 |
+
aiosignal==1.4.0
|
| 5 |
+
annotated-doc==0.0.4
|
| 6 |
+
annotated-types==0.7.0
|
| 7 |
+
antlr4-python3-runtime==4.9.3
|
| 8 |
+
anyio==4.14.1
|
| 9 |
+
async-timeout==5.0.1
|
| 10 |
+
attrs==26.1.0
|
| 11 |
+
av==13.1.0
|
| 12 |
+
bidict==0.23.1
|
| 13 |
+
blinker==1.9.0
|
| 14 |
+
certifi==2026.6.17
|
| 15 |
+
cffi==2.0.0
|
| 16 |
+
charset-normalizer==3.4.7
|
| 17 |
+
click==8.4.2
|
| 18 |
+
clip==1.0
|
| 19 |
+
coloredlogs==15.0.1
|
| 20 |
+
contourpy==1.3.2
|
| 21 |
+
cryptography==49.0.0
|
| 22 |
+
cuda-bindings==13.3.1
|
| 23 |
+
cuda-pathfinder==1.5.6
|
| 24 |
+
cuda-toolkit==13.0.2
|
| 25 |
+
cycler==0.12.1
|
| 26 |
+
dashscope==1.26.2
|
| 27 |
+
diffusers==0.31.0
|
| 28 |
+
dominate==2.9.1
|
| 29 |
+
easydict==1.13
|
| 30 |
+
einops==0.8.2
|
| 31 |
+
exceptiongroup==1.3.1
|
| 32 |
+
filelock==3.29.0
|
| 33 |
+
flash_attn==2.8.3.post1
|
| 34 |
+
Flask==3.1.3
|
| 35 |
+
Flask-SocketIO==5.6.1
|
| 36 |
+
flatbuffers==25.12.19
|
| 37 |
+
fonttools==4.63.0
|
| 38 |
+
frozenlist==1.8.0
|
| 39 |
+
fsspec==2026.4.0
|
| 40 |
+
ftfy==6.3.1
|
| 41 |
+
gitdb==4.0.12
|
| 42 |
+
GitPython==3.1.50
|
| 43 |
+
h11==0.16.0
|
| 44 |
+
hf-xet==1.5.1
|
| 45 |
+
httpcore==1.0.9
|
| 46 |
+
httpx==0.28.1
|
| 47 |
+
httpx-sse==0.4.3
|
| 48 |
+
huggingface_hub==1.21.0
|
| 49 |
+
humanfriendly==10.0
|
| 50 |
+
idna==3.18
|
| 51 |
+
ImageIO==2.37.3
|
| 52 |
+
imageio-ffmpeg==0.6.0
|
| 53 |
+
importlib_metadata==9.0.0
|
| 54 |
+
itsdangerous==2.2.0
|
| 55 |
+
Jinja2==3.1.6
|
| 56 |
+
kiwisolver==1.5.0
|
| 57 |
+
lazy-loader==0.5
|
| 58 |
+
lmdb==2.2.1
|
| 59 |
+
Mako==1.3.12
|
| 60 |
+
markdown-it-py==4.2.0
|
| 61 |
+
MarkupSafe==3.0.3
|
| 62 |
+
matplotlib==3.10.9
|
| 63 |
+
mdurl==0.1.2
|
| 64 |
+
ml_dtypes==0.5.4
|
| 65 |
+
mpmath==1.3.0
|
| 66 |
+
multidict==6.7.1
|
| 67 |
+
networkx==3.4.2
|
| 68 |
+
ninja==1.13.0
|
| 69 |
+
numpy==1.24.4
|
| 70 |
+
nvidia-cublas==13.1.1.3
|
| 71 |
+
nvidia-cublas-cu12==12.1.3.1
|
| 72 |
+
nvidia-cuda-cupti==13.0.85
|
| 73 |
+
nvidia-cuda-cupti-cu12==12.1.105
|
| 74 |
+
nvidia-cuda-nvrtc==13.0.88
|
| 75 |
+
nvidia-cuda-nvrtc-cu12==12.1.105
|
| 76 |
+
nvidia-cuda-runtime==13.0.96
|
| 77 |
+
nvidia-cuda-runtime-cu12==12.1.105
|
| 78 |
+
nvidia-cudnn-cu12==9.1.0.70
|
| 79 |
+
nvidia-cudnn-cu13==9.20.0.48
|
| 80 |
+
nvidia-cufft==12.0.0.61
|
| 81 |
+
nvidia-cufft-cu12==11.0.2.54
|
| 82 |
+
nvidia-cufile==1.15.1.6
|
| 83 |
+
nvidia-curand==10.4.0.35
|
| 84 |
+
nvidia-curand-cu12==10.3.2.106
|
| 85 |
+
nvidia-cusolver==12.0.4.66
|
| 86 |
+
nvidia-cusolver-cu12==11.4.5.107
|
| 87 |
+
nvidia-cusparse==12.6.3.3
|
| 88 |
+
nvidia-cusparse-cu12==12.1.0.106
|
| 89 |
+
nvidia-cusparselt-cu13==0.8.1
|
| 90 |
+
nvidia-ml-py==13.610.43
|
| 91 |
+
nvidia-nccl-cu12==2.21.5
|
| 92 |
+
nvidia-nccl-cu13==2.29.7
|
| 93 |
+
nvidia-nvjitlink==13.0.88
|
| 94 |
+
nvidia-nvjitlink-cu12==12.9.86
|
| 95 |
+
nvidia-nvshmem-cu13==3.4.5
|
| 96 |
+
nvidia-nvtx==13.0.85
|
| 97 |
+
nvidia-nvtx-cu12==12.1.105
|
| 98 |
+
nvidia-tensorrt==99.0.0
|
| 99 |
+
omegaconf==2.3.1
|
| 100 |
+
onnx==1.22.0
|
| 101 |
+
onnx-ir==0.2.1
|
| 102 |
+
onnxconverter-common==1.16.0
|
| 103 |
+
onnxruntime==1.23.2
|
| 104 |
+
onnxscript==0.7.1
|
| 105 |
+
open_clip_torch==3.3.0
|
| 106 |
+
opencv-python==4.11.0.86
|
| 107 |
+
orjson==3.11.9
|
| 108 |
+
packaging==26.2
|
| 109 |
+
pillow==12.2.0
|
| 110 |
+
pip==26.1.2
|
| 111 |
+
platformdirs==4.10.0
|
| 112 |
+
prettytable==3.18.0
|
| 113 |
+
propcache==0.5.2
|
| 114 |
+
protobuf==6.33.6
|
| 115 |
+
psutil==7.2.2
|
| 116 |
+
pwinput==1.0.3
|
| 117 |
+
pycocotools==2.0.11
|
| 118 |
+
pycparser==3.0
|
| 119 |
+
pycuda==2026.1
|
| 120 |
+
pydantic==2.10.6
|
| 121 |
+
pydantic_core==2.27.2
|
| 122 |
+
pydantic-settings==2.14.2
|
| 123 |
+
pyecharts==2.1.0
|
| 124 |
+
Pygments==2.20.0
|
| 125 |
+
pyparsing==3.3.2
|
| 126 |
+
python-dateutil==2.9.0.post0
|
| 127 |
+
python-dotenv==1.2.2
|
| 128 |
+
python-engineio==4.13.3
|
| 129 |
+
python-socketio==5.16.3
|
| 130 |
+
pytools==2026.1.1
|
| 131 |
+
PyYAML==6.0.3
|
| 132 |
+
regex==2026.6.28
|
| 133 |
+
requests==2.34.2
|
| 134 |
+
rich==15.0.0
|
| 135 |
+
safetensors==0.8.0
|
| 136 |
+
scikit-image==0.25.2
|
| 137 |
+
scipy==1.15.3
|
| 138 |
+
self_forcing==0.0.1
|
| 139 |
+
sentencepiece==0.2.1
|
| 140 |
+
sentry-sdk==2.64.0
|
| 141 |
+
setuptools==81.0.0
|
| 142 |
+
shellingham==1.5.4
|
| 143 |
+
simple-websocket==1.1.0
|
| 144 |
+
simplejson==4.1.1
|
| 145 |
+
siphash24==1.8
|
| 146 |
+
six==1.17.0
|
| 147 |
+
smmap==5.0.3
|
| 148 |
+
starlette==1.3.1
|
| 149 |
+
swanlab==0.9.0
|
| 150 |
+
sympy==1.13.1
|
| 151 |
+
tensorrt==11.1.0.106
|
| 152 |
+
tensorrt_cu13==11.1.0.106
|
| 153 |
+
tensorrt_cu13_bindings==11.1.0.106
|
| 154 |
+
tensorrt_cu13_libs==11.1.0.106
|
| 155 |
+
tifffile==2025.5.10
|
| 156 |
+
timm==1.0.27
|
| 157 |
+
tokenizers==0.22.2
|
| 158 |
+
torch==2.5.1+cu121
|
| 159 |
+
torchao==0.17.0
|
| 160 |
+
torchvision==0.20.1+cu121
|
| 161 |
+
tqdm==4.68.3
|
| 162 |
+
transformers==5.12.1
|
| 163 |
+
triton==3.1.0
|
| 164 |
+
typer==0.25.1
|
| 165 |
+
typing_extensions==4.15.0
|
| 166 |
+
typing-inspection==0.4.2
|
| 167 |
+
urllib3==2.7.0
|
| 168 |
+
wandb==0.28.0
|
| 169 |
+
watchdog==6.0.0
|
| 170 |
+
wcwidth==0.8.2
|
| 171 |
+
websocket-client==1.9.0
|
| 172 |
+
Werkzeug==3.1.8
|
| 173 |
+
wheel==0.47.0
|
| 174 |
+
wrapt==2.2.2
|
| 175 |
+
wsproto==1.3.2
|
| 176 |
+
yarl==1.24.2
|
| 177 |
+
zipp==4.1.0
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9/files/swanlab-metadata.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"_version":2,"hardware":{"apple_silicon":null,"cpu":{"brand":"AMD EPYC 7R13 Processor","physical_count":96,"logical_count":192},"memory":{"total":1999,"total_unit":"GB"},"accelerators":[{"vendor":"nvidia","devices":[{"index":0,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":1,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":2,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":3,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":4,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":5,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":6,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"},{"index":7,"name":"NVIDIA H100 80GB HBM3","memory":80,"memory_unit":"GB"}],"version":"580.167.08","cuda_version":"13.0","cann_version":null}]},"runtime":{"os":"Linux-6.12.90-120.164.amzn2023.x86_64-x86_64-with-glibc2.34","os_pretty":"Amazon Linux 2023.12.20260611","hostname":"ip-172-31-49-199.us-west-2.compute.internal","pid":3665088,"cwd":"/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing","python_version":"3.10.20","python_verbose":"3.10.20 | packaged by conda-forge | (main, Jun 11 2026, 03:31:56) [GCC 14.3.0]","python_executable":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python","command":"/mnt/s3files/s3-us-west2-default/zoubin/cz/envs/self_forcing/bin/python -u train.py --config_path /mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/predictor_v4_stage2_onpolicy_fppp.yaml --disable-wandb --config_override predictor_v4_rollout.output_dir=/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_onpolicy_fppp_1000_seed0 --config_override predictor_v4_rollout.max_steps=2000 --config_override predictor_v4_rollout.max_cases=null --config_override predictor_v4_rollout.use_swanlab=true"},"git":{"remote_url":"https://github.com/guandeh17/Self-Forcing","branch":"main","commit":"33593df3e81fa3ec10239271dd2c100facac6de1"},"swanlab":{"version":"0.9.0","run_dir":"/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_onpolicy_fppp_1000_seed0/swanlab/run-20260726_071952-vx0szxt9"}}
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/train_config.json
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_dtype": "bfloat16",
|
| 3 |
+
"beta1": 0.9,
|
| 4 |
+
"beta2": 0.95,
|
| 5 |
+
"blocks_lr": 5e-06,
|
| 6 |
+
"cases_path": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0/cases.jsonl",
|
| 7 |
+
"data_root": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0",
|
| 8 |
+
"default_config": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/default_config.yaml",
|
| 9 |
+
"dmd_config": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/configs/self_forcing_dmd.yaml",
|
| 10 |
+
"exact_online_timesteps": [
|
| 11 |
+
1000.0,
|
| 12 |
+
937.5,
|
| 13 |
+
833.3333129882812,
|
| 14 |
+
625.0
|
| 15 |
+
],
|
| 16 |
+
"flow_loss_weight": 1.0,
|
| 17 |
+
"fusion_lr": 5e-05,
|
| 18 |
+
"global_batch_trajectories": 8,
|
| 19 |
+
"grad_clip": 1.0,
|
| 20 |
+
"hidden_loss_weight": 0.1,
|
| 21 |
+
"keep_snapshots": 20,
|
| 22 |
+
"log_every": 1,
|
| 23 |
+
"max_cases": null,
|
| 24 |
+
"max_steps": 2000,
|
| 25 |
+
"num_workers": 0,
|
| 26 |
+
"offline_target_timestep_semantics": null,
|
| 27 |
+
"output_dir": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_stage2_onpolicy_fppp_1000_seed0",
|
| 28 |
+
"parameter_groups": {
|
| 29 |
+
"feature_fusion.anchor_norm.bias": "fusion_residual",
|
| 30 |
+
"feature_fusion.anchor_norm.weight": "fusion_residual",
|
| 31 |
+
"feature_fusion.current_norm.bias": "fusion_residual",
|
| 32 |
+
"feature_fusion.current_norm.weight": "fusion_residual",
|
| 33 |
+
"feature_fusion.mlp.0.bias": "fusion_residual",
|
| 34 |
+
"feature_fusion.mlp.0.weight": "fusion_residual",
|
| 35 |
+
"feature_fusion.mlp.2.bias": "fusion_residual",
|
| 36 |
+
"feature_fusion.mlp.2.weight": "fusion_residual",
|
| 37 |
+
"feature_fusion.previous_norm.bias": "fusion_residual",
|
| 38 |
+
"feature_fusion.previous_norm.weight": "fusion_residual",
|
| 39 |
+
"predictor_blocks.0.cross_attn.norm_q.weight": "blocks",
|
| 40 |
+
"predictor_blocks.0.cross_attn.o.bias": "blocks",
|
| 41 |
+
"predictor_blocks.0.cross_attn.o.weight": "blocks",
|
| 42 |
+
"predictor_blocks.0.cross_attn.q.bias": "blocks",
|
| 43 |
+
"predictor_blocks.0.cross_attn.q.weight": "blocks",
|
| 44 |
+
"predictor_blocks.0.ffn.0.bias": "blocks",
|
| 45 |
+
"predictor_blocks.0.ffn.0.weight": "blocks",
|
| 46 |
+
"predictor_blocks.0.ffn.2.bias": "blocks",
|
| 47 |
+
"predictor_blocks.0.ffn.2.weight": "blocks",
|
| 48 |
+
"predictor_blocks.0.modulation": "blocks",
|
| 49 |
+
"predictor_blocks.0.norm3.bias": "blocks",
|
| 50 |
+
"predictor_blocks.0.norm3.weight": "blocks",
|
| 51 |
+
"predictor_blocks.0.self_attn.k.bias": "blocks",
|
| 52 |
+
"predictor_blocks.0.self_attn.k.weight": "blocks",
|
| 53 |
+
"predictor_blocks.0.self_attn.norm_k.weight": "blocks",
|
| 54 |
+
"predictor_blocks.0.self_attn.norm_q.weight": "blocks",
|
| 55 |
+
"predictor_blocks.0.self_attn.o.bias": "blocks",
|
| 56 |
+
"predictor_blocks.0.self_attn.o.weight": "blocks",
|
| 57 |
+
"predictor_blocks.0.self_attn.q.bias": "blocks",
|
| 58 |
+
"predictor_blocks.0.self_attn.q.weight": "blocks",
|
| 59 |
+
"predictor_blocks.0.self_attn.v.bias": "blocks",
|
| 60 |
+
"predictor_blocks.0.self_attn.v.weight": "blocks",
|
| 61 |
+
"predictor_blocks.1.cross_attn.norm_q.weight": "blocks",
|
| 62 |
+
"predictor_blocks.1.cross_attn.o.bias": "blocks",
|
| 63 |
+
"predictor_blocks.1.cross_attn.o.weight": "blocks",
|
| 64 |
+
"predictor_blocks.1.cross_attn.q.bias": "blocks",
|
| 65 |
+
"predictor_blocks.1.cross_attn.q.weight": "blocks",
|
| 66 |
+
"predictor_blocks.1.ffn.0.bias": "blocks",
|
| 67 |
+
"predictor_blocks.1.ffn.0.weight": "blocks",
|
| 68 |
+
"predictor_blocks.1.ffn.2.bias": "blocks",
|
| 69 |
+
"predictor_blocks.1.ffn.2.weight": "blocks",
|
| 70 |
+
"predictor_blocks.1.modulation": "blocks",
|
| 71 |
+
"predictor_blocks.1.norm3.bias": "blocks",
|
| 72 |
+
"predictor_blocks.1.norm3.weight": "blocks",
|
| 73 |
+
"predictor_blocks.1.self_attn.k.bias": "blocks",
|
| 74 |
+
"predictor_blocks.1.self_attn.k.weight": "blocks",
|
| 75 |
+
"predictor_blocks.1.self_attn.norm_k.weight": "blocks",
|
| 76 |
+
"predictor_blocks.1.self_attn.norm_q.weight": "blocks",
|
| 77 |
+
"predictor_blocks.1.self_attn.o.bias": "blocks",
|
| 78 |
+
"predictor_blocks.1.self_attn.o.weight": "blocks",
|
| 79 |
+
"predictor_blocks.1.self_attn.q.bias": "blocks",
|
| 80 |
+
"predictor_blocks.1.self_attn.q.weight": "blocks",
|
| 81 |
+
"predictor_blocks.1.self_attn.v.bias": "blocks",
|
| 82 |
+
"predictor_blocks.1.self_attn.v.weight": "blocks",
|
| 83 |
+
"residual_out.bias": "fusion_residual",
|
| 84 |
+
"residual_out.weight": "fusion_residual"
|
| 85 |
+
},
|
| 86 |
+
"persistent_cache_semantics": "final_clean_full_pass_only",
|
| 87 |
+
"persistent_workers": false,
|
| 88 |
+
"pin_memory": false,
|
| 89 |
+
"resume": null,
|
| 90 |
+
"save_every": 100,
|
| 91 |
+
"source_block_ids": [
|
| 92 |
+
1,
|
| 93 |
+
28
|
| 94 |
+
],
|
| 95 |
+
"stage1_expected_step": 2000,
|
| 96 |
+
"stage1_training_state": "/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0/training_latest.pt",
|
| 97 |
+
"strict_teacher_load": true,
|
| 98 |
+
"supervised_states_per_optimizer_step": 144,
|
| 99 |
+
"supervision_mode": "onpolicy",
|
| 100 |
+
"swanlab_description": "Self-Forcing Predictor-v4 Stage 2 FPPP rollout with same-state online Full-Teacher targets",
|
| 101 |
+
"swanlab_experiment": "predictor-v4-stage2-onpolicy-detached-chunk-timestep-fppp-prompts1000-seed0",
|
| 102 |
+
"swanlab_mode": "cloud",
|
| 103 |
+
"swanlab_project": "Self-Forcing-Predictor-v4",
|
| 104 |
+
"swanlab_run_id": "vx0szxt9",
|
| 105 |
+
"swanlab_tags": [
|
| 106 |
+
"predictor-v4",
|
| 107 |
+
"stage2",
|
| 108 |
+
"fppp-rollout",
|
| 109 |
+
"detached-timestep",
|
| 110 |
+
"detached-chunk",
|
| 111 |
+
"onpolicy",
|
| 112 |
+
"blocks-1-28",
|
| 113 |
+
"8gpu",
|
| 114 |
+
"seed0"
|
| 115 |
+
],
|
| 116 |
+
"swanlab_workspace": null,
|
| 117 |
+
"teacher_checkpoint": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/checkpoints/self_forcing_dmd.pt",
|
| 118 |
+
"teacher_checkpoint_key": "generator_ema",
|
| 119 |
+
"train_predictor_steps": [
|
| 120 |
+
1,
|
| 121 |
+
2,
|
| 122 |
+
3
|
| 123 |
+
],
|
| 124 |
+
"trainable_parameter_dtype": "float32",
|
| 125 |
+
"trainable_parameters": 104684032,
|
| 126 |
+
"trainer_semantics": "chunk0_ffff_chunks1-6_fppp_detached_timestep_and_chunk",
|
| 127 |
+
"use_swanlab": true,
|
| 128 |
+
"warmup_steps": 100,
|
| 129 |
+
"weight_decay": 0.01,
|
| 130 |
+
"world_size": 8
|
| 131 |
+
}
|
checkpoints/predictor_v4_stage2_onpolicy_fppp_1000_seed0/train_log.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
checkpoints/predictor_v4_stage2_suite/run_suite.pid
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3663795
|
checkpoints/predictor_v4_stage2_suite/tmux_session
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
self_forcing_v4_stage2
|
checkpoints/predictor_v4_stage2_suite/wait_and_train.lock
ADDED
|
File without changes
|
checkpoints/predictor_v4_stage2_suite/wait_and_train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/self_forcing_predictor_v4_1000_seed0/cases.jsonl
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
data/self_forcing_predictor_v4_1000_seed0/dataset_config.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"inference": {
|
| 3 |
+
"candidate_block_ids": [
|
| 4 |
+
0,
|
| 5 |
+
1,
|
| 6 |
+
28,
|
| 7 |
+
29
|
| 8 |
+
],
|
| 9 |
+
"frames_per_chunk": 3,
|
| 10 |
+
"num_chunks": 7,
|
| 11 |
+
"num_steps": 4,
|
| 12 |
+
"reset_seed_per_case": true,
|
| 13 |
+
"seed": 0,
|
| 14 |
+
"teacher_checkpoint": "checkpoints/self_forcing_dmd.pt",
|
| 15 |
+
"teacher_checkpoint_key": "generator_ema"
|
| 16 |
+
},
|
| 17 |
+
"schema_version": "self_forcing_predictor_v4_bf16_v1",
|
| 18 |
+
"selection": {
|
| 19 |
+
"algorithm": "random.Random(0).sample",
|
| 20 |
+
"eligible_count": 248221,
|
| 21 |
+
"holdout": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/prompts/MovieGenVideoBench_extended.txt",
|
| 22 |
+
"holdout_exact_prompt_exclusion_count": 100,
|
| 23 |
+
"holdout_nonempty_prefix_count": 100,
|
| 24 |
+
"num_prompts": 1000,
|
| 25 |
+
"sampling_seed": 0,
|
| 26 |
+
"source": "/mnt/s3files/s3-us-west2-default/zoubin/cz/projects/Self-Forcing/prompts/vidprom_filtered_extended.txt",
|
| 27 |
+
"source_nonempty_count": 248221
|
| 28 |
+
}
|
| 29 |
+
}
|
data/self_forcing_predictor_v4_1000_seed0/manifest_summary.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"all_chunk_records": 7000,
|
| 3 |
+
"allow_incomplete": false,
|
| 4 |
+
"chunk0_excluded_from_training": true,
|
| 5 |
+
"complete_cases": 1000,
|
| 6 |
+
"expected_cases": 1000,
|
| 7 |
+
"history_storage": "incremental clean files; history_clean_prefeature_files lists chunks [0, j)",
|
| 8 |
+
"incomplete_cases": 0,
|
| 9 |
+
"training_chunk_records": 6000,
|
| 10 |
+
"training_pairs": 18000,
|
| 11 |
+
"training_pairs_per_chunk": 3
|
| 12 |
+
}
|
data/self_forcing_predictor_v4_1000_seed0/selected_prompts.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/0-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d6e6b1f1efa1e080bb9399c68d1a210833f9588b08e60ed5923a82d6ab567c9c
|
| 3 |
+
size 1377162
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/1-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:57ef460a171309292fbc90631003886a3fe11e2c4a5a81bbedcf108350c82c0b
|
| 3 |
+
size 696196
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/11-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ed6aa75acdce8ecf8901b9556dd59007f5f07a13353051ff1631c8a33774f755
|
| 3 |
+
size 3935123
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/12-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:11cc27363d24d49a84dcc3ea2f1ae21816ca4ef7261706bdd9490cc66f00c987
|
| 3 |
+
size 1842961
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/15-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0837f4d788d7b46686ae2006b013540f31cf75c4d305b678890af71a5d288d4b
|
| 3 |
+
size 1417517
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/16-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b3ca212e687c762a549471b376ef48926df1a1da7a6f219b2126270ec50d3632
|
| 3 |
+
size 1092610
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/18-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:613cef5243be8ad813395d9f463f1fe54a7341b9808ab46e7477316b53e7b81f
|
| 3 |
+
size 1580197
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/20-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a5cb1dcec4ce53dbb930e9b30bd0a5d28304b5ed1f1427db45034b97900f3f76
|
| 3 |
+
size 2588430
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/26-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7f32d629077b1ee37ac4994f56d59523341bc46bce7d1d9e2f6e4518a2f514b7
|
| 3 |
+
size 1287730
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/30-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4681424ad1229512f8a99759986fd086b9f8d5661a0e924a7ae1dd6f0c19791f
|
| 3 |
+
size 1517632
|
videos/moviegenvideobench_extended_101_200_dmd_frrf_seed0/32-0_ema.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4b18927ccd880accf280f0478b6287995adc3af4d858f09173e410cd4c2e6c4e
|
| 3 |
+
size 1449164
|